- def _add_batch_to_download_queue(self, result):
- self._log("result = %r" % (result,))
- self._log("deque = %r" % (self._deque,))
- self._deque.extend(result)
- self._log("deque after = %r" % (self._deque,))
- self._count('objects_queued', len(result))
-
- def _filter_scan_batch(self, result):
- self._log("_filter_scan_batch")
- extension = [] # consider whether this should be a dict
- for relpath_u in self._download_scan_batch.keys():
- if relpath_u in self._pending:
- continue
- file_node, metadata = max(self._download_scan_batch[relpath_u], key=lambda x: x[1]['version'])
- if self._should_download(relpath_u, metadata['version']):
- extension += [(relpath_u, file_node, metadata)]
- else:
- self._log("Excluding %r" % (relpath_u,))
- self._count('objects_excluded')
- self._call_hook(None, 'processed')
- return extension
+ def _filter_batch_to_deque(ign):
+ self._log("deque = %r, scan_batch = %r" % (self._deque, scan_batch))
+ for relpath_u in scan_batch.keys():
+ file_node, metadata = max(scan_batch[relpath_u], key=lambda x: x[1]['version'])
+
+ if self._should_download(relpath_u, metadata['version']):
+ self._deque.append( (relpath_u, file_node, metadata) )
+ else:
+ self._log("Excluding %r" % (relpath_u,))
+ self._count('objects_excluded')
+ self._call_hook(None, 'processed')
+
+ self._log("deque after = %r" % (self._deque,))
+ d.addCallback(_filter_batch_to_deque)
+ return d