added a max number of fetchers

git-svn-id: svn+ssh://svn.gna.org/svn/feed2imap/trunk/feed2imap@130 f70e237a-67f3-0310-a06c-d2b8a7116972

1 files changed, 14 insertions, 0 deletions

diff --git a/lib/feed2imap/feed2imap.rb b/lib/feed2imap/feed2imap.rb
index 5b27673..c46ff3b 100644
--- a/lib/feed2imap/feed2imap.rb
+++ b/lib/feed2imap/feed2imap.rb
@@ -102,6 +102,9 @@ class Feed2Imap
     @logger.info("Fetching and filtering feeds ...")
     ths = []
     mutex = Mutex::new
+    sparefetchers = 8 # max number of fetchers running at the same time.
+    sparefetchers_mutex = Mutex::new
+    sparefetchers_cond = ConditionVariable::new
     @config.feeds.each do |f|
       ths << Thread::new(f) do |feed|
         begin
@@ -109,6 +112,13 @@ class Feed2Imap
           lastcheck = @cache.get_last_check(feed.name) 
           if feed.needfetch(lastcheck)
             mutex.unlock
+            sparefetchers_mutex.synchronize do
+              if sparefetchers <= 0
+                sparefetchers_cond.wait(sparefetchers_mutex)
+              else
+                sparefetchers -= 1
+              end
+            end
             if feed.url
               s = HTTPFetcher::fetch(feed.url, @cache.get_last_check(feed.name))
             elsif feed.execurl
@@ -123,6 +133,10 @@ class Feed2Imap
                 s = stdout.read
               end
             end
+            sparefetchers_mutex.synchronize do
+              sparefetchers += 1
+              sparefetchers_cond.signal
+            end
             mutex.lock
             feed.body = s
             @cache.set_last_check(feed.name, Time::now)