added a max number of fetchers

git-svn-id: svn+ssh://svn.gna.org/svn/feed2imap/trunk/feed2imap@130 f70e237a-67f3-0310-a06c-d2b8a7116972

2 files changed, 17 insertions, 0 deletions

diff --git a/ChangeLog b/ChangeLog
index 18cec8d..e254402 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,9 @@ Feed2Imap 0.9.2 (XX/10/2007)
 * resynchronized rubyimap.rb with stdlib. Might fix Debian bug #405070.
 * upload items in reverse order, to upload the older first
   Closes Gna bug #8986. Thanks go do Rial Juan for the patch.
+* Don't allow more than 8 fetchers to run at the same time.
+  8 should be a reasonable default for everybody.
+  Closes Gna #9032.
 
 Feed2Imap 0.9.1 (15/05/2007)
 ============================
diff --git a/lib/feed2imap/feed2imap.rb b/lib/feed2imap/feed2imap.rb
index 5b27673..c46ff3b 100644
--- a/lib/feed2imap/feed2imap.rb
+++ b/lib/feed2imap/feed2imap.rb
@@ -102,6 +102,9 @@ class Feed2Imap
     @logger.info("Fetching and filtering feeds ...")
     ths = []
     mutex = Mutex::new
+    sparefetchers = 8 # max number of fetchers running at the same time.
+    sparefetchers_mutex = Mutex::new
+    sparefetchers_cond = ConditionVariable::new
     @config.feeds.each do |f|
       ths << Thread::new(f) do |feed|
         begin
@@ -109,6 +112,13 @@ class Feed2Imap
           lastcheck = @cache.get_last_check(feed.name) 
           if feed.needfetch(lastcheck)
             mutex.unlock
+            sparefetchers_mutex.synchronize do
+              if sparefetchers <= 0
+                sparefetchers_cond.wait(sparefetchers_mutex)
+              else
+                sparefetchers -= 1
+              end
+            end
             if feed.url
               s = HTTPFetcher::fetch(feed.url, @cache.get_last_check(feed.name))
             elsif feed.execurl
@@ -123,6 +133,10 @@ class Feed2Imap
                 s = stdout.read
               end
             end
+            sparefetchers_mutex.synchronize do
+              sparefetchers += 1
+              sparefetchers_cond.signal
+            end
             mutex.lock
             feed.body = s
             @cache.set_last_check(feed.name, Time::now)