MemoryRandomOverusedBackend used in grequests example.

scrapinghub · Feb 10, 2015 · 5a5c9e3 · 5a5c9e3
1 parent dcbfbb1
commit 5a5c9e3
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/crawlfrontier/contrib/backends/memory/__init__.py b/crawlfrontier/contrib/backends/memory/__init__.py
@@ -113,6 +113,18 @@ def get_next_requests(self, max_n_requests, downloader_info):
         return self._buffer.get_next_requests(max_n_requests, downloader_info)
 
 
+class MemoryRandomOverusedBackend(MemoryRandomBackend):
+    component_name = 'Random Memory Backend taking into account overused slots'
+
+    def __init__(self, manager):
+        super(MemoryRandomOverusedBackend, self).__init__(manager)
+        self._buffer = OverusedBuffer(super(MemoryRandomOverusedBackend, self).get_next_requests,
+                                      manager.logger.manager.debug)
+
+    def get_next_requests(self, max_n_requests, downloader_info):
+        return self._buffer.get_next_requests(max_n_requests, downloader_info)
+
+
 BASE = MemoryBaseBackend
 FIFO = MemoryFIFOBackend
 LIFO = MemoryLIFOBackend