From ddb2f1a6e32a1aae50c0f4b9bca6934814dce783 Mon Sep 17 00:00:00 2001
From: Brian Warner <warner@allmydata.com>
Date: Thu, 12 Mar 2009 21:22:29 -0700
Subject: [PATCH] consolidator: fix cycle detection to not trigger on merely
 shared directories, add snapshot counter to progress

---
 src/allmydata/scripts/consolidate.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/scripts/consolidate.py b/src/allmydata/scripts/consolidate.py
index 63c4405c..335bfeb8 100644
--- a/src/allmydata/scripts/consolidate.py
+++ b/src/allmydata/scripts/consolidate.py
@@ -42,7 +42,6 @@ class Consolidator:
         except sqlite.OperationalError, e:
             if "table dirhashes already exists" not in str(e):
                 raise
-        self.visited = set()
 
     def read_directory_json(self, dircap):
         url = self.nodeurl + "uri/%s?t=json" % urllib.quote(dircap)
@@ -180,7 +179,7 @@ class Consolidator:
         snapshots = snapshots[:-1]
 
         first_snapshot = True
-        for (timestamp, rwname, writecap, roname, readcap) in snapshots:
+        for i,(timestamp, rwname, writecap, roname, readcap) in enumerate(snapshots):
             start_created = self.directories_created
             start_used_as_is = self.directories_used_as_is
             start_reused = self.directories_reused
@@ -224,7 +223,7 @@ class Consolidator:
                 # for the others, we must scan their contents and build up a new
                 # readonly directory (which shares common subdirs with previous
                 # backups)
-                self.msg(" %s: processing" % rwname)
+                self.msg(" %s: processing (%d/%d)" % (rwname, i+1, len(snapshots)))
                 readcap = self.process_directory(readonly(writecap), (rwname,))
             if self.options["really"]:
                 self.msg("  replaced %s" % rwname)
@@ -335,14 +334,17 @@ class Consolidator:
             raiseHTTPError("error during set_children", resp)
         return dircap
 
-    def scan_old_directory(self, dircap):
+    def scan_old_directory(self, dircap, ancestors=()):
         # scan this directory (recursively) and stash a hash of its contents
         # in the DB. This assumes that all subdirs can be used as-is (i.e.
         # they've already been declared immutable)
         dircap = readonly(dircap)
-        if dircap in self.visited:
+        if dircap in ancestors:
             raise CycleDetected
-        self.visited.add(dircap)
+        ancestors = ancestors + (dircap,)
+        #self.visited.add(dircap)
+        # TODO: we could use self.visited as a mapping from dircap to dirhash,
+        # to avoid re-scanning old shared directories multiple times
         self.directories_seen.add(dircap)
         self.directories_used.add(dircap)
         data = self.read_directory_json(dircap)
@@ -350,7 +352,7 @@ class Consolidator:
         for (childname, (childtype, childdata)) in data["children"].items():
             childcap = str(childdata["ro_uri"])
             if childtype == "dirnode":
-                self.scan_old_directory(childcap)
+                self.scan_old_directory(childcap, ancestors)
             kids.append( (childname, childcap) )
         dirhash = self.hash_directory_contents(kids)
         self.store_dirhash(dirhash, dircap)
-- 
2.45.2