Verifier: check the full cryptext-hash tree on each share. Removed .todos

from the last few test_repairer tests that were waiting on this.

diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py
index cbdeb70266280c6983f2c99b5fedccaf945479cb..6e4b89e052e7cd9d46295098b395e4282371f5eb 100644 (file)
--- a/src/allmydata/immutable/checker.py
+++ b/src/allmydata/immutable/checker.py
@@ -159,6 +159,11 @@ class Checker(log.PrefixingLogMixin):
             # block fetches won't send redundant share-hash-tree requests, to
             # speed things up. Then we fetch+validate all the blockhashes.
             d.addCallback(lambda ign: vrbp.get_all_blockhashes())
+
+            cht = IncompleteHashTree(vup.num_segments)
+            cht.set_hashes({0: vup.crypttext_root_hash})
+            d.addCallback(lambda ign: vrbp.get_all_crypttext_hashes(cht))
+
             d.addCallback(lambda ign: vrbp)
             return d
         d.addCallback(_got_ueb)

diff --git a/src/allmydata/immutable/download.py b/src/allmydata/immutable/download.py
index e86bf5c4c4bf06e306cc3e466c514dfe7ba2ef64..3dde40f1f0ac2883af6f607c33c3f34819f0f269 100644 (file)
--- a/src/allmydata/immutable/download.py
+++ b/src/allmydata/immutable/download.py
@@ -419,6 +419,33 @@ class ValidatedReadBucketProxy(log.PrefixingLogMixin):
         d.addCallback(_got_block_hashes)
         return d
 
+    def get_all_crypttext_hashes(self, crypttext_hash_tree):
+        """Retrieve and validate all the crypttext-hash-tree nodes that are
+        in this share. Normally we don't look at these at all: the download
+        process fetches them incrementally as needed to validate each segment
+        of ciphertext. But this is a convenient place to give the Verifier a
+        function to validate all of these at once.
+
+        Call this with a new hashtree object for each share, initialized with
+        the crypttext hash tree root. I return a Deferred which errbacks upon
+        failure, probably with BadOrMissingHash.
+        """
+
+        # get_crypttext_hashes() always returns everything
+        d = self.bucket.get_crypttext_hashes()
+        def _got_crypttext_hashes(hashes):
+            if len(hashes) < len(crypttext_hash_tree):
+                raise BadOrMissingHash()
+            ct_hashes = dict(enumerate(hashes))
+            try:
+                crypttext_hash_tree.set_hashes(ct_hashes)
+            except IndexError, le:
+                raise BadOrMissingHash(le)
+            except (hashtree.BadHashError, hashtree.NotEnoughHashesError), le:
+                raise BadOrMissingHash(le)
+        d.addCallback(_got_crypttext_hashes)
+        return d
+
     def get_block(self, blocknum):
         # the first time we use this bucket, we need to fetch enough elements
         # of the share hash tree to validate it from our share hash up to the

diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py
index 248d93e1d8c91099688c76ac329ecbf1952f99f3..59886b2536d5ca5b47b2d10036b9d433ffd75c08 100644 (file)
--- a/src/allmydata/test/test_repairer.py
+++ b/src/allmydata/test/test_repairer.py
@@ -252,7 +252,6 @@ class Verifier(GridTestMixin, unittest.TestCase, RepairTestMixin):
         self.basedir = "repairer/Verify/truncate_crypttext_hashtree"
         return self._help_test_verify(common._corrupt_offset_of_block_hashes_to_truncate_crypttext_hashes,
                                       self.judge_invisible_corruption)
-    test_truncate_crypttext_hashtree.todo = "Verifier doesn't yet properly detect this kind of corruption."
 
     def test_corrupt_block_hashtree_offset(self):
         self.basedir = "repairer/Verify/corrupt_block_hashtree_offset"
@@ -273,13 +272,11 @@ class Verifier(GridTestMixin, unittest.TestCase, RepairTestMixin):
         self.basedir = "repairer/Verify/corrupt_crypttext_hashtree_offset"
         return self._help_test_verify(common._corrupt_offset_of_ciphertext_hash_tree,
                                       self.judge_invisible_corruption)
-    test_corrupt_crypttext_hashtree_offset.todo = "Verifier doesn't yet properly detect this kind of corruption."
 
     def test_corrupt_crypttext_hashtree(self):
         self.basedir = "repairer/Verify/corrupt_crypttext_hashtree"
         return self._help_test_verify(common._corrupt_crypttext_hash_tree,
                                       self.judge_invisible_corruption)
-    test_corrupt_crypttext_hashtree.todo = "Verifier doesn't yet properly detect this kind of corruption."
 
     def test_corrupt_block_hashtree(self):
         self.basedir = "repairer/Verify/corrupt_block_hashtree"