From 9dad69d19da7a7196ae7358e89d528999a6cea28 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Mon, 10 Jan 2011 05:24:05 -0800 Subject: httpfetcher: accept gzip encoding from servers On supported servers, gzip encoding saves bandwidth and should be enabled on clients by default to avoid excessive bandwidth bills. "deflate" encoding could be enabled, too, but servers and clients tend to handle zlib headers (or lack thereof) inconsistently and it gets messy. --- lib/feed2imap/httpfetcher.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/feed2imap/httpfetcher.rb b/lib/feed2imap/httpfetcher.rb index 2438994..9980578 100644 --- a/lib/feed2imap/httpfetcher.rb +++ b/lib/feed2imap/httpfetcher.rb @@ -17,6 +17,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA =end +require 'zlib' require 'net/http' # get openssl if available begin @@ -61,11 +62,14 @@ class HTTPFetcher useragent = 'Feed2Imap http://home.gna.org/feed2imap/' end - if lastcheck == Time::at(0) - req = Net::HTTP::Get::new(uri.request_uri, {'User-Agent' => useragent }) - else - req = Net::HTTP::Get::new(uri.request_uri, {'User-Agent' => useragent, 'If-Modified-Since' => lastcheck.httpdate}) + headers = { + 'User-Agent' => useragent, + 'Accept-Encoding' => 'gzip', + } + if lastcheck != Time::at(0) + headers.merge!('If-Modified-Since' => lastcheck.httpdate) end + req = Net::HTTP::Get::new(uri.request_uri, headers) if uri.userinfo login, pw = uri.userinfo.split(':') req.basic_auth(login, pw) @@ -81,7 +85,12 @@ class HTTPFetcher end case response when Net::HTTPSuccess - return response.body + case response['Content-Encoding'] + when 'gzip' + return Zlib::GzipReader.new(StringIO.new(response.body)).read + else + return response.body + end when Net::HTTPRedirection # if not modified if Net::HTTPNotModified === response -- cgit v1.2.3