From: Zooko O'Whielacronx <zooko@zooko.com>
Date: Fri, 30 Mar 2007 01:12:35 +0000 (-0700)
Subject: change HashTree to use new hashutil convenience methods, thus fixing a security flaw
X-Git-Url: https://git.rkrishnan.org/vdrive/quickstart.html?a=commitdiff_plain;h=c70b4773d36dfad57a8829ed541d0f3045e7df8f;p=tahoe-lafs%2Ftahoe-lafs.git

change HashTree to use new hashutil convenience methods, thus fixing a security flaw
---

diff --git a/src/allmydata/chunk.py b/src/allmydata/chunk.py
index 3ee903e5..5454b65f 100644
--- a/src/allmydata/chunk.py
+++ b/src/allmydata/chunk.py
@@ -51,11 +51,9 @@ import sha
 import os
 #import os.path
 
-from allmydata.util import bencode
+from allmydata.util.hashutil import tagged_hash, tagged_pair_hash
 
-__all__ = ['CompleteChunkFile', 'PartialChunkFile']
-
-__version__ = '1.0.0'
+__version__ = '1.0.0-allmydata'
 
 BLOCK_SIZE     = 65536
 MAX_CHUNK_SIZE = BLOCK_SIZE + 4096
@@ -171,557 +169,20 @@ class HashTree(CompleteBinaryTreeMixin, list):
     then this is used as the bottom row of the hash tree.
 
     The augmenting is done so that if the augmented element is at
-    index C{i}, then its value is C{hash(bencode.bencode((i, '')))}.
+    index C{i}, then its value is C{hash(tagged_hash('Merkle tree empty leaf', '%d'%i))}.
     """
     # Augment the list.
     start = len(L)
     end   = roundup_pow2(len(L))
     L     = L + [None] * (end - start)
     for i in range(start, end):
-      L[i] = hash(bencode.bencode((i, '')))
+      L[i] = tagged_hash('Merkle tree empty leaf', "%d"%i)
     # Form each row of the tree.
     rows = [L]
     while len(rows[-1]) != 1:
       last = rows[-1]
-      rows += [[hash(last[2*i] + last[2*i+1]) for i in xrange(len(last)//2)]]
+      rows += [[tagged_pair_hash('Merkle tree internal node', last[2*i], last[2*i+1]) for i in xrange(len(last)//2)]]
     # Flatten the list of rows into a single list.
     rows.reverse()
     self[:] = sum(rows, [])
 
-
-class BlockFile:
-  """
-  Reads and writes blocks of data to a binary file.
-
-  It is assumed that the binary file does not change in size.
-
-  @ivar file_name:    Full path to file.
-  @ivar file_size:    Size of file in bytes.
-  @ivar block_size:   Size of each block.
-  """
-  def __init__(self, file_name, mode, block_size, file_size=None):
-    """
-    Initialize block reader or writer on given file name.
-
-    If mode is 'r', the file must already exist and it is opened for
-    reading only.  If mode is 'w', the file will be created with size
-    C{file_size} if it does not exist, and it is opened for reading
-    and writing.
-
-    Note that C{file_size} is ignored if the file already exists.
-    """
-    self.mode = mode
-    self.file_name = os.path.abspath(file_name)
-    assert self.mode in ['r', 'w']
-
-    if mode == 'r':
-      f = open(self.file_name, 'rb')
-      f.close()
-
-    # Create file if it doesn't exist.
-    created = False
-    if mode == 'w' and not os.path.exists(self.file_name):
-      created = True
-      buf = ' ' * 1024
-      f = open(self.file_name, 'wb')
-      for i in xrange(file_size // len(buf)):
-        f.write(buf)
-      f.write(' ' * (file_size % len(buf)))
-      f.close()
-
-    self.file_size = os.stat(self.file_name).st_size
-    if created:
-      assert self.file_size == file_size
-    self.block_size = block_size
-    self.__block_count = self.file_size // self.block_size
-    if self.file_size % self.block_size == 0:
-      self.last_block_size = self.block_size
-    else:
-      self.last_block_size = self.file_size % self.block_size
-      self.__block_count += 1
-
-  def __getitem__(self, i):
-    """
-    Get block i.
-    """
-    if i < 0 or i >= len(self):
-      raise IndexError('block index out of range: ' + repr(i))
-    f = open(self.file_name, 'rb')
-    try:
-      f.seek(i * self.block_size)
-      ans = f.read(self.block_size)
-    finally:
-      f.close()
-    return ans
-
-  def __setitem__(self, i, s):
-    """
-    Set block i.
-    """
-    if self.mode != 'w':
-      raise ValueError('file opened for reading only')
-    if i < 0 or i >= len(self):
-      raise IndexError('block index out of range: ' + repr(i))
-    if i < len(self) - 1:
-      if len(s) != self.block_size:
-        raise ValueError('length of value must equal block_size')
-    else:
-      if len(s) != self.last_block_size:
-        raise ValueError('length of value must equal last_block_size')
-    f = open(self.file_name, 'rb+')
-    try:
-      f.seek(i * self.block_size)
-      f.write(s)
-    finally:
-      f.close()
-
-  def __len__(self):
-    """
-    Get number of blocks.
-    """
-    return int(self.__block_count)
-
-
-class MetaFile(CompleteBinaryTreeMixin):
-  """
-  A L{HashTree} stored on disk, with a timestamp.
-
-  The list of hashes can be accessed using subscripting and
-  C{__len__}, in the same manner as for L{HashTree}.
-
-  Note that the constructor takes the entire list associated with
-  the L{HashTree}, not just the bottom row of the tree.
-
-  @ivar meta_name: Full path to metafile.
-  """
-  def __init__(self, meta_name, mode, L=None):
-    """
-    Open an existing meta-file for reading or writing.
-
-    If C{mode} is 'r', the meta-file must already exist and it is
-    opened for reading only, and the list C{L} is ignored.  If C{mode}
-    is 'w', the file will be created if it does not exist (from the
-    list of hashes given in C{L}), and it is opened for reading and
-    writing.
-    """
-    self.meta_name = os.path.abspath(meta_name)
-    self.mode = mode
-    assert self.mode in ['r', 'w']
-
-    # A timestamp is stored at index 0.  The MetaFile instance
-    # offsets all indices passed to __getitem__, __setitem__ by
-    # this offset, and pretends it has length equal to
-    # self.sublength.
-    self.offset = 1
-
-    if self.mode == 'w':
-      suggested_length = len(hash('')) * (len(L)+self.offset)
-    else:
-      suggested_length = None
-
-    created = False
-    if self.mode == 'w' and not os.path.exists(self.meta_name):
-      created = True
-
-    self.block_file = BlockFile(self.meta_name, self.mode,
-                                len(hash('')),
-                                suggested_length)
-    self.sublength = len(self.block_file) - self.offset
-
-    if created:
-      for i in xrange(len(L)):
-        self.block_file[i + self.offset] = L[i]
-
-  def __getitem__(self, i):
-    if i < 0 or i >= self.sublength:
-      raise IndexError('bad meta-file block index')
-    return self.block_file[i + self.offset]
-
-  def __setitem__(self, i, value):
-    if i < 0 or i >= self.sublength:
-      raise IndexError('bad meta-file block index')
-    self.block_file[i + self.offset] = value
-
-  def __len__(self):
-    return self.sublength
-
-  def set_timestamp(self, file_name):
-    """
-    Set meta file's timestamp equal to the timestamp for C{file_name}.
-    """
-    st = os.stat(file_name)
-    timestamp = bencode.bencode((st.st_size, st.st_mtime))
-    self.block_file[0] = sha.new(timestamp).digest()
-
-  def check_timestamp(self, file_name):
-    """
-    True if meta file's timestamp equals timestamp for C{file_name}.
-    """
-    st = os.stat(file_name)
-    timestamp = bencode.bencode((st.st_size, st.st_mtime))
-    return self.block_file[0] == sha.new(timestamp).digest()
-
-
-class CompleteChunkFile(BlockFile):
-  """
-  Reads chunks from a fully-downloaded file.
-
-  A chunk C{i} is created from block C{i}.  Block C{i} is unencoded
-  data read from the file by the L{BlockFile}.  Chunk C{i} is
-  an encoded string created from block C{i}.
-
-  Chunks can be read using list subscripting.  The total number of
-  chunks (equals the total number of blocks) is given by L{__len__}.
-
-  @ivar file_name: Full path to file.
-  @ivar file_size: Size of file in bytes.
-  @ivar file_hash: Hash of file.
-  @ivar meta_name: Full path to metafile, or C{None}.
-  @ivar tree:      L{HashTree} or L{MetaFile} instance for the file.
-                   One can extract a hash from any node in the hash
-                   tree.
-  """
-
-  def __init__(self, file_name, meta_name=None, callback=None):
-    """
-    Initialize reader on the given file name.
-
-    The entire file will be read and the hash will be computed from
-    the file.  This may take a long time, so C{callback()} is called
-    frequently during this process.  This allows you to reduce CPU
-    usage if you wish.
-
-    The C{meta_name} argument is optional.  If it is specified, then the
-    hashes for C{file_name} will be stored under the file
-    C{meta_name}.  If a C{CompleteChunkFile} is created on the same
-    file and metafile in the future, then the hashes will not need to
-    be recomputed and the constructor will return instantly.  The
-    metafile contains a file and date stamp, so that if the file stored
-    in C{file_name} is modified, then the hashes will be recomputed.
-    """
-    BlockFile.__init__(self, file_name, 'r', block_size=65536)
-
-    # Whether we need to compute the hash tree
-    compute_tree = False
-
-    self.meta_name = meta_name
-    if self.meta_name != None:
-      self.meta_name = os.path.abspath(self.meta_name)
-    self.meta = None
-    if self.meta_name == None:
-      compute_tree = True
-    else:
-      try:
-        meta = MetaFile(self.meta_name, 'r')
-        assert meta.check_timestamp(self.file_name)
-      except (IOError, AssertionError):
-        compute_tree = True
-
-    # Compute the hash tree if needed.
-    if compute_tree:
-      chunk_hashes = [None] * len(self)
-      for i in xrange(len(self)):
-        triple = (self.file_size, i, BlockFile.__getitem__(self, i))
-        chunk_hashes[i] = hash(bencode.bencode(triple))
-        if callback:
-          callback()
-      self.tree = HashTree(chunk_hashes)
-      del chunk_hashes
-
-    # If a meta-file was given, make self.tree be a MetaFile instance.
-    if self.meta_name != None:
-      if compute_tree:
-        # Did we compute the hash tree?  Then store it to disk.
-        self.tree = MetaFile(self.meta_name, 'w', self.tree)
-        # Update its timestamp to be consistent with the file we
-        # just hashed.
-        self.tree.set_timestamp(self.file_name)
-      else:
-        # Read existing file from disk.
-        self.tree = MetaFile(self.meta_name, 'r')
-
-    self.file_hash = self.tree[0]
-
-  def __getitem__(self, i):
-    """
-    Get chunk C{i}.
-
-    Raises C{ValueError} if the file's contents changed since the
-    CompleteFileChunkReader was instantiated.
-    """
-    return encode_chunk(BlockFile.__getitem__(self, i), i,
-                        self.file_size, self.tree)
-
-
-def encode_chunk(block, index, file_size, tree):
-  """
-  Encode a chunk.
-
-  Given a block at index C{index} in a file with size C{file_size},
-  and a L{HashTree} or L{MetaFile} instance C{tree}, computes and
-  returns a chunk string for the given block.
-
-  The C{tree} argument needs to have correct hashes only at certain
-  indices.  Check out the code for details.  In any case, if a hash
-  is wrong an exception will be raised.
-  """
-  block_count = (len(tree) + 1) // 2
-  if index < 0 or index >= block_count:
-    raise IndexError('block index out of range: ' + repr(index))
-
-  suffix = bencode.bencode((file_size, index, block))
-  current = len(tree) - block_count + index
-  prefix = []
-  while current > 0:
-    sibling = tree.sibling(current)
-    prefix += [tree[current], tree[sibling]]
-    current = tree.parent(current)      
-  prefix = ''.join(prefix)
-
-  # Encode the chunk
-  chunk = bencode.bencode((prefix, suffix))
-
-  # Check to make sure it decodes properly.
-  decode_chunk(chunk, file_size, tree)
-  return chunk
-
-
-def decode_chunk(chunk, file_size, tree):
-  """
-  Decode a chunk.
-
-  Given file with size C{file_size} and a L{HashTree} or L{MetaFile}
-  instance C{tree}, return C{(index, block, tree_items)}.  Here
-  C{index} is the block index where string C{block} should be placed
-  in the file.  Also C{tree_items} is a dict mapping indices within
-  the L{HashTree} or L{MetaFile} tree object associated with the
-  given file to the corresponding hashes at those indices.  These
-  have been verified against the file's hash, so it is known that
-  they are correct.
-
-  Raises C{ValueError} if chunk verification fails.
-  """
-  file_hash   = tree[0]
-  block_count = (len(tree) + 1) // 2
-  try:
-    # Decode the chunk
-    try:
-      (prefix, suffix) = bencode.bdecode(chunk)
-    except:
-      raise AssertionError()
-
-    assert isinstance(prefix, str)
-    assert isinstance(suffix, str)
-
-    # Verify the suffix against the hashes in the prefix.
-    hash_len = len(hash(''))
-    L = [prefix[hash_len*i:hash_len*(i+1)] for i in range(len(prefix)//hash_len)]
-    L += [file_hash]
-    assert L[0] == hash(suffix)
-    branches = []
-    for i in range(0, len(L)-1, 2):
-      if hash(L[i] + L[i+1]) == L[i+2]:
-        branches += [0]
-      elif hash(L[i+1] + L[i]) == L[i+2]:
-        branches += [1]
-      else:
-        raise AssertionError()
-
-    # Decode the suffix
-    try:
-      (claim_file_size, claim_index, block) = bencode.bdecode(suffix)
-    except:
-      raise AssertionError()
-
-    assert isinstance(claim_file_size, int) or isinstance(claim_file_size, long)
-    assert isinstance(claim_index, int) or isinstance(claim_index, long)
-    assert isinstance(block, str)
-
-    assert file_size == claim_file_size
-
-    # Compute the index of the block, and check it.
-    found_index = sum([branches[i]*2**i for i in range(len(branches))])
-    assert found_index == claim_index
-
-    # Now fill in the tree_items dict.
-    tree_items = {}
-    current = (len(tree) - block_count) + found_index
-    i = 0
-    while current > 0 and i + 1 < len(L):
-      tree_items[current] = L[i]
-      # Next item is our sibling.
-      tree_items[tree.sibling(current)] = L[i+1]
-      i += 2
-      current = tree.parent(current)
-
-    return (found_index, block, tree_items)
-  except AssertionError:
-    raise ValueError('corrupt chunk')
-
-
-class PartialChunkFile(BlockFile):
-  """
-  Reads and writes chunks to a partially downloaded file.
-
-  @ivar file_name: Full path to file.
-  @ivar file_size: Size of file in bytes.
-  @ivar file_hash: Hash of file.
-  @ivar meta_name: Full path to metafile.
-  @ivar tree:      L{MetaFile} instance for the file.
-                   The hashes in this hash tree are valid only for
-                   nodes that we have been sent hashes for.
-  """
-  def __init__(self, file_name, meta_name, file_hash=None, file_size=None):
-    """
-    Initialize reader/writer for the given file name and metafile name.
-
-    If neither C{file_name} nor C{meta_file} exist, then both are
-    created.  The C{file_hash} and C{file_size} arguments are used to
-    initialize the two files.
-
-    If both C{file_name} and C{meta_file} exist, then the hash and
-    file size arguments are ignored, and those values are instead read
-    from the files.
-
-    If one file exists and the other does not, an C{IOError} is raised.
-    """
-    self.meta_name = os.path.abspath(meta_name)
-    meta_exists = os.path.exists(self.meta_name)
-    file_exists = os.path.exists(os.path.abspath(file_name))
-
-    BlockFile.__init__(self, os.path.abspath(file_name), 'w',
-                       BLOCK_SIZE, file_size)
-
-    if file_exists and not meta_exists:
-      raise IOError('metafile ' + repr(self.meta_name) +
-                    ' missing for file ' + repr(self.file_name))
-    if meta_exists and not file_exists:
-      raise IOError('file ' + repr(self.file_name) +
-                    ' missing for metafile ' + repr(self.meta_name))
-    tree_count = 2 * roundup_pow2(len(self)) - 1
-    self.tree = MetaFile(self.meta_name, 'w', [hash('')] * tree_count)
-
-    if not meta_exists and not file_exists:
-      self.tree[0] = file_hash
-      
-    self.file_hash = self.tree[0]
-
-  def __getitem__(self, i):
-    """
-    Get chunk C{i}.
-
-    Raises C{ValueError} if chunk has not yet been downloaded or is
-    corrupted.
-    """
-    return encode_chunk(BlockFile.__getitem__(self, i), i,
-                        self.file_size, self.tree)
-
-  def __setitem__(self, i, chunk):
-    """
-    Set chunk C{i}.
-
-    Raises C{ValueError} if the chunk is invalid.
-    """
-    (index, block, tree_items) = decode_chunk(chunk,
-                                 self.file_size, self.tree)
-    if index != i:
-      raise ValueError('incorrect index for chunk')
-    BlockFile.__setitem__(self, index, block)
-    for (tree_index, tree_value) in tree_items.items():
-      self.tree[tree_index] = tree_value
-
-
-def test(filename1='temp-out',  metaname1='temp-out.meta',
-         filename2='temp-out2', metaname2='temp-out2.meta'):
-  """
-  Unit tests.
-  """
-  print 'Testing:'
-
-  import random
-  ntests = 100
-  max_file_size = 200000
-
-  # Test CompleteChunkFile.
-
-  if os.path.exists(metaname1):
-    os.remove(metaname1)
-
-  for i in range(ntests):
-    fsize = random.randrange(max_file_size)
-    # Make some random string of size 'fsize' to go in the file.
-    s = ''.join([sha.new(str(j)).digest() for j in range(fsize//20+1)])
-    assert len(s) >= fsize
-    s = s[:fsize]
-    f = open(filename1, 'wb')
-    f.write(s)
-    f.close()
-    C = CompleteChunkFile(filename1)
-    for j in range(len(C)):
-      C[j]
-    C = CompleteChunkFile(filename1, metaname1)
-    for j in range(len(C)):
-      C[j]
-    C = CompleteChunkFile(filename1, metaname1)
-    for j in range(len(C)):
-      C[j]
-    os.remove(metaname1)
-
-  os.remove(filename1)
-
-  print '  CompleteChunkFile:      OK'
-
-  # Test PartialChunkFile
-
-  for i in range(ntests):
-    fsize = random.randrange(max_file_size)
-    # Make some random string of size 'fsize' to go in the file.
-    s = ''.join([sha.new(str(j)).digest() for j in range(fsize//20+1)])
-    assert len(s) >= fsize
-    s = s[:fsize]
-    f = open(filename1, 'wb')
-    f.write(s)
-    f.close()
-    C1 = CompleteChunkFile(filename1)
-    if os.path.exists(filename2):
-      os.remove(filename2)
-
-    if os.path.exists(metaname2):
-      os.remove(metaname2)
-    C2 = PartialChunkFile(filename2, metaname2, C1.file_hash, C1.file_size)
-    assert len(C1) == len(C2)
-    assert C2.tree[0] == C1.tree[0]
-    for j in range(len(C2)):
-      try:
-        C2[j]
-        ok = False
-      except ValueError:
-        ok = True
-      if not ok:
-        raise AssertionError()
-    for j in range(len(C2)//2):
-      k = random.randrange(len(C2))
-      if len(C1) > 1:
-        assert C1[k] != C1[(k+1)%len(C1)]
-        try:
-          C2[k] = C1[(k+1)%len(C1)]
-          ok = False
-        except ValueError:
-          ok = True
-        if not ok:
-          raise AssertionError()
-      C2[k] = C1[k]
-      assert C2[k] == C1[k]
-    for j in range(len(C2)):
-      C2[j] = C1[j]
-      assert C2[j] == C1[j]
-
-  os.remove(filename1)
-  os.remove(filename2)
-  os.remove(metaname2)
-
-  print '  PartialChunkFile:       OK'
-
-
-if __name__ == '__main__':
-  test()