src/allmydata/mutable/layout.py

   1
   2 import struct
   3 from allmydata.mutable.common import NeedMoreDataError, UnknownVersionError
   4
   5 PREFIX = ">BQ32s16s" # each version has a different prefix
   6 SIGNED_PREFIX = ">BQ32s16s BBQQ" # this is covered by the signature
   7 SIGNED_PREFIX_LENGTH = struct.calcsize(SIGNED_PREFIX)
   8 HEADER = ">BQ32s16s BBQQ LLLLQQ" # includes offsets
   9 HEADER_LENGTH = struct.calcsize(HEADER)
  10
  11 def unpack_header(data):
  12     o = {}
  13     (version,
  14      seqnum,
  15      root_hash,
  16      IV,
  17      k, N, segsize, datalen,
  18      o['signature'],
  19      o['share_hash_chain'],
  20      o['block_hash_tree'],
  21      o['share_data'],
  22      o['enc_privkey'],
  23      o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
  24     return (version, seqnum, root_hash, IV, k, N, segsize, datalen, o)
  25
  26 def unpack_prefix_and_signature(data):
  27     assert len(data) >= HEADER_LENGTH, len(data)
  28     prefix = data[:SIGNED_PREFIX_LENGTH]
  29
  30     (version,
  31      seqnum,
  32      root_hash,
  33      IV,
  34      k, N, segsize, datalen,
  35      o) = unpack_header(data)
  36
  37     if version != 0:
  38         raise UnknownVersionError("got mutable share version %d, but I only understand version 0" % version)
  39
  40     if len(data) < o['share_hash_chain']:
  41         raise NeedMoreDataError(o['share_hash_chain'],
  42                                 o['enc_privkey'], o['EOF']-o['enc_privkey'])
  43
  44     pubkey_s = data[HEADER_LENGTH:o['signature']]
  45     signature = data[o['signature']:o['share_hash_chain']]
  46
  47     return (seqnum, root_hash, IV, k, N, segsize, datalen,
  48             pubkey_s, signature, prefix)
  49
  50 def unpack_share(data):
  51     assert len(data) >= HEADER_LENGTH
  52     o = {}
  53     (version,
  54      seqnum,
  55      root_hash,
  56      IV,
  57      k, N, segsize, datalen,
  58      o['signature'],
  59      o['share_hash_chain'],
  60      o['block_hash_tree'],
  61      o['share_data'],
  62      o['enc_privkey'],
  63      o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
  64
  65     if version != 0:
  66         raise UnknownVersionError("got mutable share version %d, but I only understand version 0" % version)
  67
  68     if len(data) < o['EOF']:
  69         raise NeedMoreDataError(o['EOF'],
  70                                 o['enc_privkey'], o['EOF']-o['enc_privkey'])
  71
  72     pubkey = data[HEADER_LENGTH:o['signature']]
  73     signature = data[o['signature']:o['share_hash_chain']]
  74     share_hash_chain_s = data[o['share_hash_chain']:o['block_hash_tree']]
  75     share_hash_format = ">H32s"
  76     hsize = struct.calcsize(share_hash_format)
  77     assert len(share_hash_chain_s) % hsize == 0, len(share_hash_chain_s)
  78     share_hash_chain = []
  79     for i in range(0, len(share_hash_chain_s), hsize):
  80         chunk = share_hash_chain_s[i:i+hsize]
  81         (hid, h) = struct.unpack(share_hash_format, chunk)
  82         share_hash_chain.append( (hid, h) )
  83     share_hash_chain = dict(share_hash_chain)
  84     block_hash_tree_s = data[o['block_hash_tree']:o['share_data']]
  85     assert len(block_hash_tree_s) % 32 == 0, len(block_hash_tree_s)
  86     block_hash_tree = []
  87     for i in range(0, len(block_hash_tree_s), 32):
  88         block_hash_tree.append(block_hash_tree_s[i:i+32])
  89
  90     share_data = data[o['share_data']:o['enc_privkey']]
  91     enc_privkey = data[o['enc_privkey']:o['EOF']]
  92
  93     return (seqnum, root_hash, IV, k, N, segsize, datalen,
  94             pubkey, signature, share_hash_chain, block_hash_tree,
  95             share_data, enc_privkey)
  96
  97 def unpack_share_data(verinfo, hash_and_data):
  98     (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, o_t) = verinfo
  99
 100     # hash_and_data starts with the share_hash_chain, so figure out what the
 101     # offsets really are
 102     o = dict(o_t)
 103     o_share_hash_chain = 0
 104     o_block_hash_tree = o['block_hash_tree'] - o['share_hash_chain']
 105     o_share_data = o['share_data'] - o['share_hash_chain']
 106     o_enc_privkey = o['enc_privkey'] - o['share_hash_chain']
 107
 108     share_hash_chain_s = hash_and_data[o_share_hash_chain:o_block_hash_tree]
 109     share_hash_format = ">H32s"
 110     hsize = struct.calcsize(share_hash_format)
 111     assert len(share_hash_chain_s) % hsize == 0, len(share_hash_chain_s)
 112     share_hash_chain = []
 113     for i in range(0, len(share_hash_chain_s), hsize):
 114         chunk = share_hash_chain_s[i:i+hsize]
 115         (hid, h) = struct.unpack(share_hash_format, chunk)
 116         share_hash_chain.append( (hid, h) )
 117     share_hash_chain = dict(share_hash_chain)
 118     block_hash_tree_s = hash_and_data[o_block_hash_tree:o_share_data]
 119     assert len(block_hash_tree_s) % 32 == 0, len(block_hash_tree_s)
 120     block_hash_tree = []
 121     for i in range(0, len(block_hash_tree_s), 32):
 122         block_hash_tree.append(block_hash_tree_s[i:i+32])
 123
 124     share_data = hash_and_data[o_share_data:o_enc_privkey]
 125
 126     return (share_hash_chain, block_hash_tree, share_data)
 127
 128
 129 def pack_checkstring(seqnum, root_hash, IV):
 130     return struct.pack(PREFIX,
 131                        0, # version,
 132                        seqnum,
 133                        root_hash,
 134                        IV)
 135
 136 def unpack_checkstring(checkstring):
 137     cs_len = struct.calcsize(PREFIX)
 138     version, seqnum, root_hash, IV = struct.unpack(PREFIX, checkstring[:cs_len])
 139     if version != 0: # TODO: just ignore the share
 140         raise UnknownVersionError("got mutable share version %d, but I only understand version 0" % version)
 141     return (seqnum, root_hash, IV)
 142
 143 def pack_prefix(seqnum, root_hash, IV,
 144                 required_shares, total_shares,
 145                 segment_size, data_length):
 146     prefix = struct.pack(SIGNED_PREFIX,
 147                          0, # version,
 148                          seqnum,
 149                          root_hash,
 150                          IV,
 151
 152                          required_shares,
 153                          total_shares,
 154                          segment_size,
 155                          data_length,
 156                          )
 157     return prefix
 158
 159 def pack_offsets(verification_key_length, signature_length,
 160                  share_hash_chain_length, block_hash_tree_length,
 161                  share_data_length, encprivkey_length):
 162     post_offset = HEADER_LENGTH
 163     offsets = {}
 164     o1 = offsets['signature'] = post_offset + verification_key_length
 165     o2 = offsets['share_hash_chain'] = o1 + signature_length
 166     o3 = offsets['block_hash_tree'] = o2 + share_hash_chain_length
 167     o4 = offsets['share_data'] = o3 + block_hash_tree_length
 168     o5 = offsets['enc_privkey'] = o4 + share_data_length
 169     offsets['EOF'] = o5 + encprivkey_length
 170
 171     return struct.pack(">LLLLQQ",
 172                        offsets['signature'],
 173                        offsets['share_hash_chain'],
 174                        offsets['block_hash_tree'],
 175                        offsets['share_data'],
 176                        offsets['enc_privkey'],
 177                        offsets['EOF'])
 178
 179 def pack_share(prefix, verification_key, signature,
 180                share_hash_chain, block_hash_tree,
 181                share_data, encprivkey):
 182     share_hash_chain_s = "".join([struct.pack(">H32s", i, share_hash_chain[i])
 183                                   for i in sorted(share_hash_chain.keys())])
 184     for h in block_hash_tree:
 185         assert len(h) == 32
 186     block_hash_tree_s = "".join(block_hash_tree)
 187
 188     offsets = pack_offsets(len(verification_key),
 189                            len(signature),
 190                            len(share_hash_chain_s),
 191                            len(block_hash_tree_s),
 192                            len(share_data),
 193                            len(encprivkey))
 194     final_share = "".join([prefix,
 195                            offsets,
 196                            verification_key,
 197                            signature,
 198                            share_hash_chain_s,
 199                            block_hash_tree_s,
 200                            share_data,
 201                            encprivkey])
 202     return final_share
 203