src/allmydata/mutable/layout.py

   1
   2 import struct
   3 from common import NeedMoreDataError
   4
   5 PREFIX = ">BQ32s16s" # each version has a different prefix
   6 SIGNED_PREFIX = ">BQ32s16s BBQQ" # this is covered by the signature
   7 SIGNED_PREFIX_LENGTH = struct.calcsize(SIGNED_PREFIX)
   8 HEADER = ">BQ32s16s BBQQ LLLLQQ" # includes offsets
   9 HEADER_LENGTH = struct.calcsize(HEADER)
  10
  11 def unpack_header(data):
  12     o = {}
  13     (version,
  14      seqnum,
  15      root_hash,
  16      IV,
  17      k, N, segsize, datalen,
  18      o['signature'],
  19      o['share_hash_chain'],
  20      o['block_hash_tree'],
  21      o['share_data'],
  22      o['enc_privkey'],
  23      o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
  24     return (version, seqnum, root_hash, IV, k, N, segsize, datalen, o)
  25
  26 def unpack_prefix_and_signature(data):
  27     assert len(data) >= HEADER_LENGTH, len(data)
  28     prefix = data[:SIGNED_PREFIX_LENGTH]
  29
  30     (version,
  31      seqnum,
  32      root_hash,
  33      IV,
  34      k, N, segsize, datalen,
  35      o) = unpack_header(data)
  36
  37     assert version == 0
  38     if len(data) < o['share_hash_chain']:
  39         raise NeedMoreDataError(o['share_hash_chain'],
  40                                 o['enc_privkey'], o['EOF']-o['enc_privkey'])
  41
  42     pubkey_s = data[HEADER_LENGTH:o['signature']]
  43     signature = data[o['signature']:o['share_hash_chain']]
  44
  45     return (seqnum, root_hash, IV, k, N, segsize, datalen,
  46             pubkey_s, signature, prefix)
  47
  48 def unpack_share(data):
  49     assert len(data) >= HEADER_LENGTH
  50     o = {}
  51     (version,
  52      seqnum,
  53      root_hash,
  54      IV,
  55      k, N, segsize, datalen,
  56      o['signature'],
  57      o['share_hash_chain'],
  58      o['block_hash_tree'],
  59      o['share_data'],
  60      o['enc_privkey'],
  61      o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
  62
  63     assert version == 0
  64     if len(data) < o['EOF']:
  65         raise NeedMoreDataError(o['EOF'],
  66                                 o['enc_privkey'], o['EOF']-o['enc_privkey'])
  67
  68     pubkey = data[HEADER_LENGTH:o['signature']]
  69     signature = data[o['signature']:o['share_hash_chain']]
  70     share_hash_chain_s = data[o['share_hash_chain']:o['block_hash_tree']]
  71     share_hash_format = ">H32s"
  72     hsize = struct.calcsize(share_hash_format)
  73     assert len(share_hash_chain_s) % hsize == 0, len(share_hash_chain_s)
  74     share_hash_chain = []
  75     for i in range(0, len(share_hash_chain_s), hsize):
  76         chunk = share_hash_chain_s[i:i+hsize]
  77         (hid, h) = struct.unpack(share_hash_format, chunk)
  78         share_hash_chain.append( (hid, h) )
  79     share_hash_chain = dict(share_hash_chain)
  80     block_hash_tree_s = data[o['block_hash_tree']:o['share_data']]
  81     assert len(block_hash_tree_s) % 32 == 0, len(block_hash_tree_s)
  82     block_hash_tree = []
  83     for i in range(0, len(block_hash_tree_s), 32):
  84         block_hash_tree.append(block_hash_tree_s[i:i+32])
  85
  86     share_data = data[o['share_data']:o['enc_privkey']]
  87     enc_privkey = data[o['enc_privkey']:o['EOF']]
  88
  89     return (seqnum, root_hash, IV, k, N, segsize, datalen,
  90             pubkey, signature, share_hash_chain, block_hash_tree,
  91             share_data, enc_privkey)
  92
  93 def unpack_share_data(verinfo, hash_and_data):
  94     (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, o_t) = verinfo
  95
  96     # hash_and_data starts with the share_hash_chain, so figure out what the
  97     # offsets really are
  98     o = dict(o_t)
  99     o_share_hash_chain = 0
 100     o_block_hash_tree = o['block_hash_tree'] - o['share_hash_chain']
 101     o_share_data = o['share_data'] - o['share_hash_chain']
 102     o_enc_privkey = o['enc_privkey'] - o['share_hash_chain']
 103
 104     share_hash_chain_s = hash_and_data[o_share_hash_chain:o_block_hash_tree]
 105     share_hash_format = ">H32s"
 106     hsize = struct.calcsize(share_hash_format)
 107     assert len(share_hash_chain_s) % hsize == 0, len(share_hash_chain_s)
 108     share_hash_chain = []
 109     for i in range(0, len(share_hash_chain_s), hsize):
 110         chunk = share_hash_chain_s[i:i+hsize]
 111         (hid, h) = struct.unpack(share_hash_format, chunk)
 112         share_hash_chain.append( (hid, h) )
 113     share_hash_chain = dict(share_hash_chain)
 114     block_hash_tree_s = hash_and_data[o_block_hash_tree:o_share_data]
 115     assert len(block_hash_tree_s) % 32 == 0, len(block_hash_tree_s)
 116     block_hash_tree = []
 117     for i in range(0, len(block_hash_tree_s), 32):
 118         block_hash_tree.append(block_hash_tree_s[i:i+32])
 119
 120     share_data = hash_and_data[o_share_data:o_enc_privkey]
 121
 122     return (share_hash_chain, block_hash_tree, share_data)
 123
 124
 125 def pack_checkstring(seqnum, root_hash, IV):
 126     return struct.pack(PREFIX,
 127                        0, # version,
 128                        seqnum,
 129                        root_hash,
 130                        IV)
 131
 132 def unpack_checkstring(checkstring):
 133     cs_len = struct.calcsize(PREFIX)
 134     version, seqnum, root_hash, IV = struct.unpack(PREFIX, checkstring[:cs_len])
 135     assert version == 0 # TODO: just ignore the share
 136     return (seqnum, root_hash, IV)
 137
 138 def pack_prefix(seqnum, root_hash, IV,
 139                 required_shares, total_shares,
 140                 segment_size, data_length):
 141     prefix = struct.pack(SIGNED_PREFIX,
 142                          0, # version,
 143                          seqnum,
 144                          root_hash,
 145                          IV,
 146
 147                          required_shares,
 148                          total_shares,
 149                          segment_size,
 150                          data_length,
 151                          )
 152     return prefix
 153
 154 def pack_offsets(verification_key_length, signature_length,
 155                  share_hash_chain_length, block_hash_tree_length,
 156                  share_data_length, encprivkey_length):
 157     post_offset = HEADER_LENGTH
 158     offsets = {}
 159     o1 = offsets['signature'] = post_offset + verification_key_length
 160     o2 = offsets['share_hash_chain'] = o1 + signature_length
 161     o3 = offsets['block_hash_tree'] = o2 + share_hash_chain_length
 162     o4 = offsets['share_data'] = o3 + block_hash_tree_length
 163     o5 = offsets['enc_privkey'] = o4 + share_data_length
 164     o6 = offsets['EOF'] = o5 + encprivkey_length
 165
 166     return struct.pack(">LLLLQQ",
 167                        offsets['signature'],
 168                        offsets['share_hash_chain'],
 169                        offsets['block_hash_tree'],
 170                        offsets['share_data'],
 171                        offsets['enc_privkey'],
 172                        offsets['EOF'])
 173
 174 def pack_share(prefix, verification_key, signature,
 175                share_hash_chain, block_hash_tree,
 176                share_data, encprivkey):
 177     share_hash_chain_s = "".join([struct.pack(">H32s", i, share_hash_chain[i])
 178                                   for i in sorted(share_hash_chain.keys())])
 179     for h in block_hash_tree:
 180         assert len(h) == 32
 181     block_hash_tree_s = "".join(block_hash_tree)
 182
 183     offsets = pack_offsets(len(verification_key),
 184                            len(signature),
 185                            len(share_hash_chain_s),
 186                            len(block_hash_tree_s),
 187                            len(share_data),
 188                            len(encprivkey))
 189     final_share = "".join([prefix,
 190                            offsets,
 191                            verification_key,
 192                            signature,
 193                            share_hash_chain_s,
 194                            block_hash_tree_s,
 195                            share_data,
 196                            encprivkey])
 197     return final_share
 198