src/allmydata/mutable/layout.py

   1
   2 import struct
   3 from common import NeedMoreDataError
   4
   5 PREFIX = ">BQ32s16s" # each version has a different prefix
   6 SIGNED_PREFIX = ">BQ32s16s BBQQ" # this is covered by the signature
   7 HEADER = ">BQ32s16s BBQQ LLLLQQ" # includes offsets
   8 HEADER_LENGTH = struct.calcsize(HEADER)
   9
  10 def unpack_header(data):
  11     o = {}
  12     (version,
  13      seqnum,
  14      root_hash,
  15      IV,
  16      k, N, segsize, datalen,
  17      o['signature'],
  18      o['share_hash_chain'],
  19      o['block_hash_tree'],
  20      o['share_data'],
  21      o['enc_privkey'],
  22      o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
  23     return (version, seqnum, root_hash, IV, k, N, segsize, datalen, o)
  24
  25 def unpack_prefix_and_signature(data):
  26     assert len(data) >= HEADER_LENGTH, len(data)
  27     prefix = data[:struct.calcsize(SIGNED_PREFIX)]
  28
  29     (version,
  30      seqnum,
  31      root_hash,
  32      IV,
  33      k, N, segsize, datalen,
  34      o) = unpack_header(data)
  35
  36     assert version == 0
  37     if len(data) < o['share_hash_chain']:
  38         raise NeedMoreDataError(o['share_hash_chain'],
  39                                 o['enc_privkey'], o['EOF']-o['enc_privkey'])
  40
  41     pubkey_s = data[HEADER_LENGTH:o['signature']]
  42     signature = data[o['signature']:o['share_hash_chain']]
  43
  44     return (seqnum, root_hash, IV, k, N, segsize, datalen,
  45             pubkey_s, signature, prefix)
  46
  47 def unpack_share(data):
  48     assert len(data) >= HEADER_LENGTH
  49     o = {}
  50     (version,
  51      seqnum,
  52      root_hash,
  53      IV,
  54      k, N, segsize, datalen,
  55      o['signature'],
  56      o['share_hash_chain'],
  57      o['block_hash_tree'],
  58      o['share_data'],
  59      o['enc_privkey'],
  60      o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
  61
  62     assert version == 0
  63     if len(data) < o['EOF']:
  64         raise NeedMoreDataError(o['EOF'],
  65                                 o['enc_privkey'], o['EOF']-o['enc_privkey'])
  66
  67     pubkey = data[HEADER_LENGTH:o['signature']]
  68     signature = data[o['signature']:o['share_hash_chain']]
  69     share_hash_chain_s = data[o['share_hash_chain']:o['block_hash_tree']]
  70     share_hash_format = ">H32s"
  71     hsize = struct.calcsize(share_hash_format)
  72     assert len(share_hash_chain_s) % hsize == 0, len(share_hash_chain_s)
  73     share_hash_chain = []
  74     for i in range(0, len(share_hash_chain_s), hsize):
  75         chunk = share_hash_chain_s[i:i+hsize]
  76         (hid, h) = struct.unpack(share_hash_format, chunk)
  77         share_hash_chain.append( (hid, h) )
  78     share_hash_chain = dict(share_hash_chain)
  79     block_hash_tree_s = data[o['block_hash_tree']:o['share_data']]
  80     assert len(block_hash_tree_s) % 32 == 0, len(block_hash_tree_s)
  81     block_hash_tree = []
  82     for i in range(0, len(block_hash_tree_s), 32):
  83         block_hash_tree.append(block_hash_tree_s[i:i+32])
  84
  85     share_data = data[o['share_data']:o['enc_privkey']]
  86     enc_privkey = data[o['enc_privkey']:o['EOF']]
  87
  88     return (seqnum, root_hash, IV, k, N, segsize, datalen,
  89             pubkey, signature, share_hash_chain, block_hash_tree,
  90             share_data, enc_privkey)
  91
  92 def unpack_share_data(verinfo, hash_and_data):
  93     (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, o_t) = verinfo
  94
  95     # hash_and_data starts with the share_hash_chain, so figure out what the
  96     # offsets really are
  97     o = dict(o_t)
  98     o_share_hash_chain = 0
  99     o_block_hash_tree = o['block_hash_tree'] - o['share_hash_chain']
 100     o_share_data = o['share_data'] - o['share_hash_chain']
 101     o_enc_privkey = o['enc_privkey'] - o['share_hash_chain']
 102
 103     share_hash_chain_s = hash_and_data[o_share_hash_chain:o_block_hash_tree]
 104     share_hash_format = ">H32s"
 105     hsize = struct.calcsize(share_hash_format)
 106     assert len(share_hash_chain_s) % hsize == 0, len(share_hash_chain_s)
 107     share_hash_chain = []
 108     for i in range(0, len(share_hash_chain_s), hsize):
 109         chunk = share_hash_chain_s[i:i+hsize]
 110         (hid, h) = struct.unpack(share_hash_format, chunk)
 111         share_hash_chain.append( (hid, h) )
 112     share_hash_chain = dict(share_hash_chain)
 113     block_hash_tree_s = hash_and_data[o_block_hash_tree:o_share_data]
 114     assert len(block_hash_tree_s) % 32 == 0, len(block_hash_tree_s)
 115     block_hash_tree = []
 116     for i in range(0, len(block_hash_tree_s), 32):
 117         block_hash_tree.append(block_hash_tree_s[i:i+32])
 118
 119     share_data = hash_and_data[o_share_data:o_enc_privkey]
 120
 121     return (share_hash_chain, block_hash_tree, share_data)
 122
 123
 124 def pack_checkstring(seqnum, root_hash, IV):
 125     return struct.pack(PREFIX,
 126                        0, # version,
 127                        seqnum,
 128                        root_hash,
 129                        IV)
 130
 131 def unpack_checkstring(checkstring):
 132     cs_len = struct.calcsize(PREFIX)
 133     version, seqnum, root_hash, IV = struct.unpack(PREFIX, checkstring[:cs_len])
 134     assert version == 0 # TODO: just ignore the share
 135     return (seqnum, root_hash, IV)
 136
 137 def pack_prefix(seqnum, root_hash, IV,
 138                 required_shares, total_shares,
 139                 segment_size, data_length):
 140     prefix = struct.pack(SIGNED_PREFIX,
 141                          0, # version,
 142                          seqnum,
 143                          root_hash,
 144                          IV,
 145
 146                          required_shares,
 147                          total_shares,
 148                          segment_size,
 149                          data_length,
 150                          )
 151     return prefix
 152
 153 def pack_offsets(verification_key_length, signature_length,
 154                  share_hash_chain_length, block_hash_tree_length,
 155                  share_data_length, encprivkey_length):
 156     post_offset = HEADER_LENGTH
 157     offsets = {}
 158     o1 = offsets['signature'] = post_offset + verification_key_length
 159     o2 = offsets['share_hash_chain'] = o1 + signature_length
 160     o3 = offsets['block_hash_tree'] = o2 + share_hash_chain_length
 161     o4 = offsets['share_data'] = o3 + block_hash_tree_length
 162     o5 = offsets['enc_privkey'] = o4 + share_data_length
 163     o6 = offsets['EOF'] = o5 + encprivkey_length
 164
 165     return struct.pack(">LLLLQQ",
 166                        offsets['signature'],
 167                        offsets['share_hash_chain'],
 168                        offsets['block_hash_tree'],
 169                        offsets['share_data'],
 170                        offsets['enc_privkey'],
 171                        offsets['EOF'])
 172
 173 def pack_share(prefix, verification_key, signature,
 174                share_hash_chain, block_hash_tree,
 175                share_data, encprivkey):
 176     share_hash_chain_s = "".join([struct.pack(">H32s", i, share_hash_chain[i])
 177                                   for i in sorted(share_hash_chain.keys())])
 178     for h in block_hash_tree:
 179         assert len(h) == 32
 180     block_hash_tree_s = "".join(block_hash_tree)
 181
 182     offsets = pack_offsets(len(verification_key),
 183                            len(signature),
 184                            len(share_hash_chain_s),
 185                            len(block_hash_tree_s),
 186                            len(share_data),
 187                            len(encprivkey))
 188     final_share = "".join([prefix,
 189                            offsets,
 190                            verification_key,
 191                            signature,
 192                            share_hash_chain_s,
 193                            block_hash_tree_s,
 194                            share_data,
 195                            encprivkey])
 196     return final_share
 197