3 from allmydata.mutable.common import NeedMoreDataError, UnknownVersionError
4 from allmydata.interfaces import HASH_SIZE, SALT_SIZE, SDMF_VERSION, \
5 MDMF_VERSION, IMutableSlotWriter
6 from allmydata.util import mathutil
7 from twisted.python import failure
8 from twisted.internet import defer
9 from zope.interface import implements
12 # These strings describe the format of the packed structs they help process
13 # Here's what they mean:
16 # >: Big-endian byte order; the most significant byte is first (leftmost).
17 # B: The version information; an 8 bit version identifier. Stored as
18 # an unsigned char. This is currently 00 00 00 00; our modifications
19 # will turn it into 00 00 00 01.
20 # Q: The sequence number; this is sort of like a revision history for
21 # mutable files; they start at 1 and increase as they are changed after
22 # being uploaded. Stored as an unsigned long long, which is 8 bytes in
24 # 32s: The root hash of the share hash tree. We use sha-256d, so we use 32
25 # characters = 32 bytes to store the value.
26 # 16s: The salt for the readkey. This is a 16-byte random value, stored as
29 # SIGNED_PREFIX additions, things that are covered by the signature:
30 # B: The "k" encoding parameter. We store this as an 8-bit character,
31 # which is convenient because our erasure coding scheme cannot
32 # encode if you ask for more than 255 pieces.
33 # B: The "N" encoding parameter. Stored as an 8-bit character for the
34 # same reasons as above.
35 # Q: The segment size of the uploaded file. This will essentially be the
36 # length of the file in SDMF. An unsigned long long, so we can store
37 # files of quite large size.
38 # Q: The data length of the uploaded file. Modulo padding, this will be
39 # the same of the data length field. Like the data length field, it is
40 # an unsigned long long and can be quite large.
43 # L: The offset of the signature of this. An unsigned long.
44 # L: The offset of the share hash chain. An unsigned long.
45 # L: The offset of the block hash tree. An unsigned long.
46 # L: The offset of the share data. An unsigned long.
47 # Q: The offset of the encrypted private key. An unsigned long long, to
48 # account for the possibility of a lot of share data.
49 # Q: The offset of the EOF. An unsigned long long, to account for the
50 # possibility of a lot of share data.
52 # After all of these, we have the following:
53 # - The verification key: Occupies the space between the end of the header
54 # and the start of the signature (i.e.: data[HEADER_LENGTH:o['signature']].
55 # - The signature, which goes from the signature offset to the share hash
57 # - The share hash chain, which goes from the share hash chain offset to
58 # the block hash tree offset.
59 # - The share data, which goes from the share data offset to the encrypted
61 # - The encrypted private key offset, which goes until the end of the file.
63 # The block hash tree in this encoding has only one share, so the offset of
64 # the share data will be 32 bits more than the offset of the block hash tree.
65 # Given this, we may need to check to see how many bytes a reasonably sized
66 # block hash tree will take up.
68 PREFIX = ">BQ32s16s" # each version has a different prefix
69 SIGNED_PREFIX = ">BQ32s16s BBQQ" # this is covered by the signature
70 SIGNED_PREFIX_LENGTH = struct.calcsize(SIGNED_PREFIX)
71 HEADER = ">BQ32s16s BBQQ LLLLQQ" # includes offsets
72 HEADER_LENGTH = struct.calcsize(HEADER)
74 OFFSETS_LENGTH = struct.calcsize(OFFSETS)
76 # These are still used for some tests.
77 def unpack_header(data):
83 k, N, segsize, datalen,
85 o['share_hash_chain'],
89 o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
90 return (version, seqnum, root_hash, IV, k, N, segsize, datalen, o)
92 def unpack_share(data):
93 assert len(data) >= HEADER_LENGTH
99 k, N, segsize, datalen,
101 o['share_hash_chain'],
102 o['block_hash_tree'],
105 o['EOF']) = struct.unpack(HEADER, data[:HEADER_LENGTH])
108 raise UnknownVersionError("got mutable share version %d, but I only understand version 0" % version)
110 if len(data) < o['EOF']:
111 raise NeedMoreDataError(o['EOF'],
112 o['enc_privkey'], o['EOF']-o['enc_privkey'])
114 pubkey = data[HEADER_LENGTH:o['signature']]
115 signature = data[o['signature']:o['share_hash_chain']]
116 share_hash_chain_s = data[o['share_hash_chain']:o['block_hash_tree']]
117 share_hash_format = ">H32s"
118 hsize = struct.calcsize(share_hash_format)
119 assert len(share_hash_chain_s) % hsize == 0, len(share_hash_chain_s)
120 share_hash_chain = []
121 for i in range(0, len(share_hash_chain_s), hsize):
122 chunk = share_hash_chain_s[i:i+hsize]
123 (hid, h) = struct.unpack(share_hash_format, chunk)
124 share_hash_chain.append( (hid, h) )
125 share_hash_chain = dict(share_hash_chain)
126 block_hash_tree_s = data[o['block_hash_tree']:o['share_data']]
127 assert len(block_hash_tree_s) % 32 == 0, len(block_hash_tree_s)
129 for i in range(0, len(block_hash_tree_s), 32):
130 block_hash_tree.append(block_hash_tree_s[i:i+32])
132 share_data = data[o['share_data']:o['enc_privkey']]
133 enc_privkey = data[o['enc_privkey']:o['EOF']]
135 return (seqnum, root_hash, IV, k, N, segsize, datalen,
136 pubkey, signature, share_hash_chain, block_hash_tree,
137 share_data, enc_privkey)
139 def unpack_checkstring(checkstring):
140 cs_len = struct.calcsize(PREFIX)
141 version, seqnum, root_hash, IV = struct.unpack(PREFIX, checkstring[:cs_len])
142 if version != 0: # TODO: just ignore the share
143 raise UnknownVersionError("got mutable share version %d, but I only understand version 0" % version)
144 return (seqnum, root_hash, IV)
147 def pack_offsets(verification_key_length, signature_length,
148 share_hash_chain_length, block_hash_tree_length,
149 share_data_length, encprivkey_length):
150 post_offset = HEADER_LENGTH
152 o1 = offsets['signature'] = post_offset + verification_key_length
153 o2 = offsets['share_hash_chain'] = o1 + signature_length
154 o3 = offsets['block_hash_tree'] = o2 + share_hash_chain_length
155 o4 = offsets['share_data'] = o3 + block_hash_tree_length
156 o5 = offsets['enc_privkey'] = o4 + share_data_length
157 offsets['EOF'] = o5 + encprivkey_length
159 return struct.pack(">LLLLQQ",
160 offsets['signature'],
161 offsets['share_hash_chain'],
162 offsets['block_hash_tree'],
163 offsets['share_data'],
164 offsets['enc_privkey'],
167 def pack_share(prefix, verification_key, signature,
168 share_hash_chain, block_hash_tree,
169 share_data, encprivkey):
170 share_hash_chain_s = "".join([struct.pack(">H32s", i, share_hash_chain[i])
171 for i in sorted(share_hash_chain.keys())])
172 for h in block_hash_tree:
174 block_hash_tree_s = "".join(block_hash_tree)
176 offsets = pack_offsets(len(verification_key),
178 len(share_hash_chain_s),
179 len(block_hash_tree_s),
182 final_share = "".join([prefix,
192 def pack_prefix(seqnum, root_hash, IV,
193 required_shares, total_shares,
194 segment_size, data_length):
195 prefix = struct.pack(SIGNED_PREFIX,
208 class SDMFSlotWriteProxy:
209 implements(IMutableSlotWriter)
211 I represent a remote write slot for an SDMF mutable file. I build a
212 share in memory, and then write it in one piece to the remote
213 server. This mimics how SDMF shares were built before MDMF (and the
214 new MDMF uploader), but provides that functionality in a way that
215 allows the MDMF uploader to be built without much special-casing for
216 file format, which makes the uploader code more readable.
220 rref, # a remote reference to a storage server
222 secrets, # (write_enabler, renew_secret, cancel_secret)
223 seqnum, # the sequence number of the mutable file
227 data_length): # the length of the original file
230 self._storage_index = storage_index
231 self._secrets = secrets
232 self._seqnum = seqnum
233 self._required_shares = required_shares
234 self._total_shares = total_shares
235 self._segment_size = segment_size
236 self._data_length = data_length
238 # This is an SDMF file, so it should have only one segment, so,
239 # modulo padding of the data length, the segment size and the
240 # data length should be the same.
241 expected_segment_size = mathutil.next_multiple(data_length,
242 self._required_shares)
243 assert expected_segment_size == segment_size
245 self._block_size = self._segment_size / self._required_shares
247 # This is meant to mimic how SDMF files were built before MDMF
248 # entered the picture: we generate each share in its entirety,
249 # then push it off to the storage server in one write. When
250 # callers call set_*, they are just populating this dict.
251 # finish_publishing will stitch these pieces together into a
252 # coherent share, and then write the coherent share to the
254 self._share_pieces = {}
256 # This tells the write logic what checkstring to use when
257 # writing remote shares.
260 self._readvs = [(0, struct.calcsize(PREFIX))]
263 def set_checkstring(self, checkstring_or_seqnum,
267 Set the checkstring that I will pass to the remote server when
270 @param checkstring_or_seqnum: A packed checkstring to use,
271 or a sequence number. I will treat this as a checkstr
273 Note that implementations can differ in which semantics they
274 wish to support for set_checkstring -- they can, for example,
275 build the checkstring themselves from its constituents, or
278 if root_hash and salt:
279 checkstring = struct.pack(PREFIX,
281 checkstring_or_seqnum,
285 checkstring = checkstring_or_seqnum
286 self._testvs = [(0, len(checkstring), "eq", checkstring)]
289 def get_checkstring(self):
291 Get the checkstring that I think currently exists on the remote
295 return self._testvs[0][3]
299 def put_block(self, data, segnum, salt):
301 Add a block and salt to the share.
303 # SDMF files have only one segment
305 assert len(data) == self._block_size
306 assert len(salt) == SALT_SIZE
308 self._share_pieces['sharedata'] = data
309 self._share_pieces['salt'] = salt
311 # TODO: Figure out something intelligent to return.
312 return defer.succeed(None)
315 def put_encprivkey(self, encprivkey):
317 Add the encrypted private key to the share.
319 self._share_pieces['encprivkey'] = encprivkey
321 return defer.succeed(None)
324 def put_blockhashes(self, blockhashes):
326 Add the block hash tree to the share.
328 assert isinstance(blockhashes, list)
329 for h in blockhashes:
330 assert len(h) == HASH_SIZE
332 # serialize the blockhashes, then set them.
333 blockhashes_s = "".join(blockhashes)
334 self._share_pieces['block_hash_tree'] = blockhashes_s
336 return defer.succeed(None)
339 def put_sharehashes(self, sharehashes):
341 Add the share hash chain to the share.
343 assert isinstance(sharehashes, dict)
344 for h in sharehashes.itervalues():
345 assert len(h) == HASH_SIZE
347 # serialize the sharehashes, then set them.
348 sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
349 for i in sorted(sharehashes.keys())])
350 self._share_pieces['share_hash_chain'] = sharehashes_s
352 return defer.succeed(None)
355 def put_root_hash(self, root_hash):
357 Add the root hash to the share.
359 assert len(root_hash) == HASH_SIZE
361 self._share_pieces['root_hash'] = root_hash
363 return defer.succeed(None)
366 def put_salt(self, salt):
368 Add a salt to an empty SDMF file.
370 assert len(salt) == SALT_SIZE
372 self._share_pieces['salt'] = salt
373 self._share_pieces['sharedata'] = ""
376 def get_signable(self):
378 Return the part of the share that needs to be signed.
380 SDMF writers need to sign the packed representation of the
381 first eight fields of the remote share, that is:
384 - root of the share hash tree
391 This method is responsible for returning that to callers.
393 return struct.pack(SIGNED_PREFIX,
396 self._share_pieces['root_hash'],
397 self._share_pieces['salt'],
398 self._required_shares,
404 def put_signature(self, signature):
406 Add the signature to the share.
408 self._share_pieces['signature'] = signature
410 return defer.succeed(None)
413 def put_verification_key(self, verification_key):
415 Add the verification key to the share.
417 self._share_pieces['verification_key'] = verification_key
419 return defer.succeed(None)
422 def get_verinfo(self):
424 I return my verinfo tuple. This is used by the ServermapUpdater
425 to keep track of versions of mutable files.
427 The verinfo tuple for MDMF files contains:
435 - prefix (the thing that you sign)
438 We include the nonce in MDMF to simplify processing of version
441 The verinfo tuple for SDMF files is the same, but contains a
442 16-byte IV instead of a hash of salts.
444 return (self._seqnum,
445 self._share_pieces['root_hash'],
446 self._share_pieces['salt'],
449 self._required_shares,
452 self._get_offsets_tuple())
454 def _get_offsets_dict(self):
455 post_offset = HEADER_LENGTH
458 verification_key_length = len(self._share_pieces['verification_key'])
459 o1 = offsets['signature'] = post_offset + verification_key_length
461 signature_length = len(self._share_pieces['signature'])
462 o2 = offsets['share_hash_chain'] = o1 + signature_length
464 share_hash_chain_length = len(self._share_pieces['share_hash_chain'])
465 o3 = offsets['block_hash_tree'] = o2 + share_hash_chain_length
467 block_hash_tree_length = len(self._share_pieces['block_hash_tree'])
468 o4 = offsets['share_data'] = o3 + block_hash_tree_length
470 share_data_length = len(self._share_pieces['sharedata'])
471 o5 = offsets['enc_privkey'] = o4 + share_data_length
473 encprivkey_length = len(self._share_pieces['encprivkey'])
474 offsets['EOF'] = o5 + encprivkey_length
478 def _get_offsets_tuple(self):
479 offsets = self._get_offsets_dict()
480 return tuple([(key, value) for key, value in offsets.items()])
483 def _pack_offsets(self):
484 offsets = self._get_offsets_dict()
485 return struct.pack(">LLLLQQ",
486 offsets['signature'],
487 offsets['share_hash_chain'],
488 offsets['block_hash_tree'],
489 offsets['share_data'],
490 offsets['enc_privkey'],
494 def finish_publishing(self):
496 Do anything necessary to finish writing the share to a remote
497 server. I require that no further publishing needs to take place
498 after this method has been called.
500 for k in ["sharedata", "encprivkey", "signature", "verification_key",
501 "share_hash_chain", "block_hash_tree"]:
502 assert k in self._share_pieces, (self.shnum, k, self._share_pieces.keys())
503 # This is the only method that actually writes something to the
505 # First, we need to pack the share into data that we can write
506 # to the remote server in one write.
507 offsets = self._pack_offsets()
508 prefix = self.get_signable()
509 final_share = "".join([prefix,
511 self._share_pieces['verification_key'],
512 self._share_pieces['signature'],
513 self._share_pieces['share_hash_chain'],
514 self._share_pieces['block_hash_tree'],
515 self._share_pieces['sharedata'],
516 self._share_pieces['encprivkey']])
518 # Our only data vector is going to be writing the final share,
520 datavs = [(0, final_share)]
523 # Our caller has not provided us with another checkstring
524 # yet, so we assume that we are writing a new share, and set
525 # a test vector that will allow a new share to be written.
527 self._testvs.append(tuple([0, 1, "eq", ""]))
530 tw_vectors[self.shnum] = (self._testvs, datavs, None)
531 return self._rref.callRemote("slot_testv_and_readv_and_writev",
535 # TODO is it useful to read something?
539 MDMFHEADER = ">BQ32sBBQQ QQQQQQQQ"
540 MDMFHEADERWITHOUTOFFSETS = ">BQ32sBBQQ"
541 MDMFHEADERSIZE = struct.calcsize(MDMFHEADER)
542 MDMFHEADERWITHOUTOFFSETSSIZE = struct.calcsize(MDMFHEADERWITHOUTOFFSETS)
543 MDMFCHECKSTRING = ">BQ32s"
544 MDMFSIGNABLEHEADER = ">BQ32sBBQQ"
545 MDMFOFFSETS = ">QQQQQQQQ"
546 MDMFOFFSETS_LENGTH = struct.calcsize(MDMFOFFSETS)
548 PRIVATE_KEY_SIZE = 1220
550 VERIFICATION_KEY_SIZE = 292
551 # We know we won't have more than 256 shares, and we know that we won't need
552 # to store more than ln2(256) hash-chain nodes to validate, so that's our
553 # bound. Each node requires 2 bytes of node-number plus 32 bytes of hash.
554 SHARE_HASH_CHAIN_SIZE = (2+HASH_SIZE)*mathutil.log_ceil(256, 2)
556 class MDMFSlotWriteProxy:
557 implements(IMutableSlotWriter)
560 I represent a remote write slot for an MDMF mutable file.
562 I abstract away from my caller the details of block and salt
563 management, and the implementation of the on-disk format for MDMF
566 # Expected layout, MDMF:
567 # offset: size: name:
569 # 0 1 version number (01)
570 # 1 8 sequence number
571 # 9 32 share tree root hash
572 # 41 1 The "k" encoding parameter
573 # 42 1 The "N" encoding parameter
574 # 43 8 The segment size of the uploaded file
575 # 51 8 The data length of the original plaintext
576 #-- end signed part --
577 # 59 8 The offset of the encrypted private key
578 # 67 8 The offset of the share hash chain
579 # 75 8 The offset of the signature
580 # 83 8 The offset of the verification key
581 # 91 8 The offset of the end of the v. key.
582 # 99 8 The offset of the share data
583 # 107 8 The offset of the block hash tree
584 # 115 8 The offset of EOF
585 # 123 var encrypted private key
586 # var var share hash chain
588 # var var verification key
589 # var large share data
590 # var var block hash tree
592 # We order the fields that way to make smart downloaders -- downloaders
593 # which prempetively read a big part of the share -- possible.
595 # The checkstring is the first three fields -- the version number,
596 # sequence number, root hash and root salt hash. This is consistent
597 # in meaning to what we have with SDMF files, except now instead of
598 # using the literal salt, we use a value derived from all of the
599 # salts -- the share hash root.
601 # The salt is stored before the block for each segment. The block
602 # hash tree is computed over the combination of block and salt for
603 # each segment. In this way, we get integrity checking for both
604 # block and salt with the current block hash tree arrangement.
606 # The ordering of the offsets is different to reflect the dependencies
607 # that we'll run into with an MDMF file. The expected write flow is
608 # something like this:
610 # 0: Initialize with the sequence number, encoding parameters and
611 # data length. From this, we can deduce the number of segments,
612 # and where they should go.. We can also figure out where the
613 # encrypted private key should go, because we can figure out how
614 # big the share data will be.
616 # 1: Encrypt, encode, and upload the file in chunks. Do something
619 # put_block(data, segnum, salt)
621 # to write a block and a salt to the disk. We can do both of
622 # these operations now because we have enough of the offsets to
623 # know where to put them.
625 # 2: Put the encrypted private key. Use:
627 # put_encprivkey(encprivkey)
629 # Now that we know the length of the private key, we can fill
630 # in the offset for the block hash tree.
632 # 3: We're now in a position to upload the block hash tree for
633 # a share. Put that using something like:
635 # put_blockhashes(block_hash_tree)
637 # Note that block_hash_tree is a list of hashes -- we'll take
638 # care of the details of serializing that appropriately. When
639 # we get the block hash tree, we are also in a position to
640 # calculate the offset for the share hash chain, and fill that
641 # into the offsets table.
643 # 4: We're now in a position to upload the share hash chain for
644 # a share. Do that with something like:
646 # put_sharehashes(share_hash_chain)
648 # share_hash_chain should be a dictionary mapping shnums to
649 # 32-byte hashes -- the wrapper handles serialization.
650 # We'll know where to put the signature at this point, also.
651 # The root of this tree will be put explicitly in the next
654 # 5: Before putting the signature, we must first put the
655 # root_hash. Do this with:
657 # put_root_hash(root_hash).
659 # In terms of knowing where to put this value, it was always
660 # possible to place it, but it makes sense semantically to
661 # place it after the share hash tree, so that's why you do it
664 # 6: With the root hash put, we can now sign the header. Use:
668 # to get the part of the header that you want to sign, and use:
670 # put_signature(signature)
672 # to write your signature to the remote server.
674 # 6: Add the verification key, and finish. Do:
676 # put_verification_key(key)
682 # Checkstring management:
684 # To write to a mutable slot, we have to provide test vectors to ensure
685 # that we are writing to the same data that we think we are. These
686 # vectors allow us to detect uncoordinated writes; that is, writes
687 # where both we and some other shareholder are writing to the
688 # mutable slot, and to report those back to the parts of the program
691 # With SDMF, this was easy -- all of the share data was written in
692 # one go, so it was easy to detect uncoordinated writes, and we only
693 # had to do it once. With MDMF, not all of the file is written at
696 # If a share is new, we write out as much of the header as we can
697 # before writing out anything else. This gives other writers a
698 # canary that they can use to detect uncoordinated writes, and, if
699 # they do the same thing, gives us the same canary. We them update
700 # the share. We won't be able to write out two fields of the header
701 # -- the share tree hash and the salt hash -- until we finish
702 # writing out the share. We only require the writer to provide the
703 # initial checkstring, and keep track of what it should be after
706 # If we haven't written anything yet, then on the first write (which
707 # will probably be a block + salt of a share), we'll also write out
708 # the header. On subsequent passes, we'll expect to see the header.
709 # This changes in two places:
711 # - When we write out the salt hash
712 # - When we write out the root of the share hash tree
714 # since these values will change the header. It is possible that we
715 # can just make those be written in one operation to minimize
719 rref, # a remote reference to a storage server
721 secrets, # (write_enabler, renew_secret, cancel_secret)
722 seqnum, # the sequence number of the mutable file
726 data_length): # the length of the original file
729 self._storage_index = storage_index
730 self._seqnum = seqnum
731 self._required_shares = required_shares
732 assert self.shnum >= 0 and self.shnum < total_shares
733 self._total_shares = total_shares
734 # We build up the offset table as we write things. It is the
735 # last thing we write to the remote server.
738 # This is a list of write vectors that will be sent to our
739 # remote server once we are directed to write things there.
741 self._secrets = secrets
742 # The segment size needs to be a multiple of the k parameter --
743 # any padding should have been carried out by the publisher
745 assert segment_size % required_shares == 0
746 self._segment_size = segment_size
747 self._data_length = data_length
749 # These are set later -- we define them here so that we can
750 # check for their existence easily
752 # This is the root of the share hash tree -- the Merkle tree
753 # over the roots of the block hash trees computed for shares in
755 self._root_hash = None
757 # We haven't yet written anything to the remote bucket. By
758 # setting this, we tell the _write method as much. The write
759 # method will then know that it also needs to add a write vector
760 # for the checkstring (or what we have of it) to the first write
761 # request. We'll then record that value for future use. If
762 # we're expecting something to be there already, we need to call
763 # set_checkstring before we write anything to tell the first
765 self._written = False
767 # When writing data to the storage servers, we get a read vector
768 # for free. We'll read the checkstring, which will help us
769 # figure out what's gone wrong if a write fails.
770 self._readv = [(0, struct.calcsize(MDMFCHECKSTRING))]
772 # We calculate the number of segments because it tells us
773 # where the salt part of the file ends/share segment begins,
774 # and also because it provides a useful amount of bounds checking.
775 self._num_segments = mathutil.div_ceil(self._data_length,
777 self._block_size = self._segment_size / self._required_shares
778 # We also calculate the share size, to help us with block
780 tail_size = self._data_length % self._segment_size
782 self._tail_block_size = self._block_size
784 self._tail_block_size = mathutil.next_multiple(tail_size,
785 self._required_shares)
786 self._tail_block_size /= self._required_shares
788 # We already know where the sharedata starts; right after the end
789 # of the header (which is defined as the signable part + the offsets)
790 # We can also calculate where the encrypted private key begins
791 # from what we know know.
792 self._actual_block_size = self._block_size + SALT_SIZE
793 data_size = self._actual_block_size * (self._num_segments - 1)
794 data_size += self._tail_block_size
795 data_size += SALT_SIZE
796 self._offsets['enc_privkey'] = MDMFHEADERSIZE
798 # We don't define offsets for these because we want them to be
799 # tightly packed -- this allows us to ignore the responsibility
800 # of padding individual values, and of removing that padding
801 # later. So nonconstant_start is where we start writing
803 nonconstant_start = self._offsets['enc_privkey']
804 nonconstant_start += PRIVATE_KEY_SIZE
805 nonconstant_start += SIGNATURE_SIZE
806 nonconstant_start += VERIFICATION_KEY_SIZE
807 nonconstant_start += SHARE_HASH_CHAIN_SIZE
809 self._offsets['share_data'] = nonconstant_start
811 # Finally, we know how big the share data will be, so we can
812 # figure out where the block hash tree needs to go.
813 # XXX: But this will go away if Zooko wants to make it so that
814 # you don't need to know the size of the file before you start
816 self._offsets['block_hash_tree'] = self._offsets['share_data'] + \
819 # Done. We can snow start writing.
822 def set_checkstring(self,
823 seqnum_or_checkstring,
827 Set checkstring checkstring for the given shnum.
829 This can be invoked in one of two ways.
831 With one argument, I assume that you are giving me a literal
832 checkstring -- e.g., the output of get_checkstring. I will then
833 set that checkstring as it is. This form is used by unit tests.
835 With two arguments, I assume that you are giving me a sequence
836 number and root hash to make a checkstring from. In that case, I
837 will build a checkstring and set it for you. This form is used
840 By default, I assume that I am writing new shares to the grid.
841 If you don't explcitly set your own checkstring, I will use
842 one that requires that the remote share not exist. You will want
843 to use this method if you are updating a share in-place;
844 otherwise, writes will fail.
846 # You're allowed to overwrite checkstrings with this method;
847 # I assume that users know what they are doing when they call
850 checkstring = struct.pack(MDMFCHECKSTRING,
852 seqnum_or_checkstring,
855 checkstring = seqnum_or_checkstring
857 if checkstring == "":
858 # We special-case this, since len("") = 0, but we need
859 # length of 1 for the case of an empty share to work on the
860 # storage server, which is what a checkstring that is the
861 # empty string means.
865 self._testvs.append((0, len(checkstring), "eq", checkstring))
869 return "MDMFSlotWriteProxy for share %d" % self.shnum
872 def get_checkstring(self):
874 Given a share number, I return a representation of what the
875 checkstring for that share on the server will look like.
877 I am mostly used for tests.
880 roothash = self._root_hash
882 roothash = "\x00" * 32
883 return struct.pack(MDMFCHECKSTRING,
889 def put_block(self, data, segnum, salt):
891 I queue a write vector for the data, salt, and segment number
892 provided to me. I return None, as I do not actually cause
893 anything to be written yet.
895 if segnum >= self._num_segments:
896 raise LayoutInvalid("I won't overwrite the block hash tree")
897 if len(salt) != SALT_SIZE:
898 raise LayoutInvalid("I was given a salt of size %d, but "
899 "I wanted a salt of size %d")
900 if segnum + 1 == self._num_segments:
901 if len(data) != self._tail_block_size:
902 raise LayoutInvalid("I was given the wrong size block to write")
903 elif len(data) != self._block_size:
904 raise LayoutInvalid("I was given the wrong size block to write")
906 # We want to write at len(MDMFHEADER) + segnum * block_size.
907 offset = self._offsets['share_data'] + \
908 (self._actual_block_size * segnum)
911 self._writevs.append(tuple([offset, data]))
914 def put_encprivkey(self, encprivkey):
916 I queue a write vector for the encrypted private key provided to
920 assert self._offsets['enc_privkey']
921 # You shouldn't re-write the encprivkey after the block hash
922 # tree is written, since that could cause the private key to run
923 # into the block hash tree. Before it writes the block hash
924 # tree, the block hash tree writing method writes the offset of
925 # the share hash chain. So that's a good indicator of whether or
926 # not the block hash tree has been written.
927 if "signature" in self._offsets:
928 raise LayoutInvalid("You can't put the encrypted private key "
929 "after putting the share hash chain")
931 self._offsets['share_hash_chain'] = self._offsets['enc_privkey'] + \
934 self._writevs.append(tuple([self._offsets['enc_privkey'], encprivkey]))
937 def put_blockhashes(self, blockhashes):
939 I queue a write vector to put the block hash tree in blockhashes
940 onto the remote server.
942 The encrypted private key must be queued before the block hash
943 tree, since we need to know how large it is to know where the
944 block hash tree should go. The block hash tree must be put
945 before the share hash chain, since its size determines the
946 offset of the share hash chain.
949 assert "block_hash_tree" in self._offsets
951 assert isinstance(blockhashes, list)
953 blockhashes_s = "".join(blockhashes)
954 self._offsets['EOF'] = self._offsets['block_hash_tree'] + len(blockhashes_s)
956 self._writevs.append(tuple([self._offsets['block_hash_tree'],
960 def put_sharehashes(self, sharehashes):
962 I queue a write vector to put the share hash chain in my
963 argument onto the remote server.
965 The block hash tree must be queued before the share hash chain,
966 since we need to know where the block hash tree ends before we
967 can know where the share hash chain starts. The share hash chain
968 must be put before the signature, since the length of the packed
969 share hash chain determines the offset of the signature. Also,
970 semantically, you must know what the root of the block hash tree
971 is before you can generate a valid signature.
973 assert isinstance(sharehashes, dict)
975 if "share_hash_chain" not in self._offsets:
976 raise LayoutInvalid("You must put the block hash tree before "
977 "putting the share hash chain")
979 # The signature comes after the share hash chain. If the
980 # signature has already been written, we must not write another
981 # share hash chain. The signature writes the verification key
982 # offset when it gets sent to the remote server, so we look for
984 if "verification_key" in self._offsets:
985 raise LayoutInvalid("You must write the share hash chain "
986 "before you write the signature")
987 sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
988 for i in sorted(sharehashes.keys())])
989 self._offsets['signature'] = self._offsets['share_hash_chain'] + \
991 self._writevs.append(tuple([self._offsets['share_hash_chain'],
995 def put_root_hash(self, roothash):
997 Put the root hash (the root of the share hash tree) in the
1000 # It does not make sense to be able to put the root
1001 # hash without first putting the share hashes, since you need
1002 # the share hashes to generate the root hash.
1004 # Signature is defined by the routine that places the share hash
1005 # chain, so it's a good thing to look for in finding out whether
1006 # or not the share hash chain exists on the remote server.
1007 if len(roothash) != HASH_SIZE:
1008 raise LayoutInvalid("hashes and salts must be exactly %d bytes"
1010 self._root_hash = roothash
1011 # To write both of these values, we update the checkstring on
1012 # the remote server, which includes them
1013 checkstring = self.get_checkstring()
1014 self._writevs.append(tuple([0, checkstring]))
1015 # This write, if successful, changes the checkstring, so we need
1016 # to update our internal checkstring to be consistent with the
1017 # one on the server.
1020 def get_signable(self):
1022 Get the first seven fields of the mutable file; the parts that
1025 if not self._root_hash:
1026 raise LayoutInvalid("You need to set the root hash "
1027 "before getting something to "
1029 return struct.pack(MDMFSIGNABLEHEADER,
1033 self._required_shares,
1039 def put_signature(self, signature):
1041 I queue a write vector for the signature of the MDMF share.
1043 I require that the root hash and share hash chain have been put
1044 to the grid before I will write the signature to the grid.
1046 if "signature" not in self._offsets:
1047 raise LayoutInvalid("You must put the share hash chain "
1048 # It does not make sense to put a signature without first
1049 # putting the root hash and the salt hash (since otherwise
1050 # the signature would be incomplete), so we don't allow that.
1051 "before putting the signature")
1052 if not self._root_hash:
1053 raise LayoutInvalid("You must complete the signed prefix "
1054 "before computing a signature")
1055 # If we put the signature after we put the verification key, we
1056 # could end up running into the verification key, and will
1057 # probably screw up the offsets as well. So we don't allow that.
1058 if "verification_key_end" in self._offsets:
1059 raise LayoutInvalid("You can't put the signature after the "
1061 # The method that writes the verification key defines the EOF
1062 # offset before writing the verification key, so look for that.
1063 self._offsets['verification_key'] = self._offsets['signature'] +\
1065 self._writevs.append(tuple([self._offsets['signature'], signature]))
1068 def put_verification_key(self, verification_key):
1070 I queue a write vector for the verification key.
1072 I require that the signature have been written to the storage
1073 server before I allow the verification key to be written to the
1076 if "verification_key" not in self._offsets:
1077 raise LayoutInvalid("You must put the signature before you "
1078 "can put the verification key")
1080 self._offsets['verification_key_end'] = \
1081 self._offsets['verification_key'] + len(verification_key)
1082 assert self._offsets['verification_key_end'] <= self._offsets['share_data']
1083 self._writevs.append(tuple([self._offsets['verification_key'],
1087 def _get_offsets_tuple(self):
1088 return tuple([(key, value) for key, value in self._offsets.items()])
1091 def get_verinfo(self):
1092 return (self._seqnum,
1094 self._required_shares,
1098 self.get_signable(),
1099 self._get_offsets_tuple())
1102 def finish_publishing(self):
1104 I add a write vector for the offsets table, and then cause all
1105 of the write vectors that I've dealt with so far to be published
1106 to the remote server, ending the write process.
1108 if "verification_key_end" not in self._offsets:
1109 raise LayoutInvalid("You must put the verification key before "
1110 "you can publish the offsets")
1111 offsets_offset = struct.calcsize(MDMFHEADERWITHOUTOFFSETS)
1112 offsets = struct.pack(MDMFOFFSETS,
1113 self._offsets['enc_privkey'],
1114 self._offsets['share_hash_chain'],
1115 self._offsets['signature'],
1116 self._offsets['verification_key'],
1117 self._offsets['verification_key_end'],
1118 self._offsets['share_data'],
1119 self._offsets['block_hash_tree'],
1120 self._offsets['EOF'])
1121 self._writevs.append(tuple([offsets_offset, offsets]))
1122 encoding_parameters_offset = struct.calcsize(MDMFCHECKSTRING)
1123 params = struct.pack(">BBQQ",
1124 self._required_shares,
1128 self._writevs.append(tuple([encoding_parameters_offset, params]))
1129 return self._write(self._writevs)
1132 def _write(self, datavs, on_failure=None, on_success=None):
1133 """I write the data vectors in datavs to the remote slot."""
1135 if not self._testvs:
1137 self._testvs.append(tuple([0, 1, "eq", ""]))
1138 if not self._written:
1139 # Write a new checkstring to the share when we write it, so
1140 # that we have something to check later.
1141 new_checkstring = self.get_checkstring()
1142 datavs.append((0, new_checkstring))
1144 self._written = True
1145 self._testvs = [(0, len(new_checkstring), "eq", new_checkstring)]
1146 on_success = _first_write
1147 tw_vectors[self.shnum] = (self._testvs, datavs, None)
1148 d = self._rref.callRemote("slot_testv_and_readv_and_writev",
1149 self._storage_index,
1153 def _result(results):
1154 if isinstance(results, failure.Failure) or not results[0]:
1155 # Do nothing; the write was unsuccessful.
1156 if on_failure: on_failure()
1158 if on_success: on_success()
1160 d.addCallback(_result)
1164 class MDMFSlotReadProxy:
1166 I read from a mutable slot filled with data written in the MDMF data
1167 format (which is described above).
1169 I can be initialized with some amount of data, which I will use (if
1170 it is valid) to eliminate some of the need to fetch it from servers.
1177 # Start the initialization process.
1179 self._storage_index = storage_index
1182 # Before doing anything, the reader is probably going to want to
1183 # verify that the signature is correct. To do that, they'll need
1184 # the verification key, and the signature. To get those, we'll
1185 # need the offset table. So fetch the offset table on the
1186 # assumption that that will be the first thing that a reader is
1189 # The fact that these encoding parameters are None tells us
1190 # that we haven't yet fetched them from the remote share, so we
1191 # should. We could just not set them, but the checks will be
1192 # easier to read if we don't have to use hasattr.
1193 self._version_number = None
1194 self._sequence_number = None
1195 self._root_hash = None
1196 # Filled in if we're dealing with an SDMF file. Unused
1199 self._required_shares = None
1200 self._total_shares = None
1201 self._segment_size = None
1202 self._data_length = None
1203 self._offsets = None
1205 # If the user has chosen to initialize us with some data, we'll
1206 # try to satisfy subsequent data requests with that data before
1207 # asking the storage server for it. If
1209 # The way callers interact with cache in the filenode returns
1210 # None if there isn't any cached data, but the way we index the
1211 # cached data requires a string, so convert None to "".
1212 if self._data == None:
1216 def _maybe_fetch_offsets_and_header(self, force_remote=False):
1218 I fetch the offset table and the header from the remote slot if
1219 I don't already have them. If I do have them, I do nothing and
1220 return an empty Deferred.
1223 return defer.succeed(None)
1224 # At this point, we may be either SDMF or MDMF. Fetching 107
1225 # bytes will be enough to get header and offsets for both SDMF and
1226 # MDMF, though we'll be left with 4 more bytes than we
1227 # need if this ends up being MDMF. This is probably less
1228 # expensive than the cost of a second roundtrip.
1230 d = self._read(readvs, force_remote)
1231 d.addCallback(self._process_encoding_parameters)
1232 d.addCallback(self._process_offsets)
1236 def _process_encoding_parameters(self, encoding_parameters):
1237 assert self.shnum in encoding_parameters
1238 encoding_parameters = encoding_parameters[self.shnum][0]
1239 # The first byte is the version number. It will tell us what
1241 (verno,) = struct.unpack(">B", encoding_parameters[:1])
1242 if verno == MDMF_VERSION:
1243 read_size = MDMFHEADERWITHOUTOFFSETSSIZE
1250 datalen) = struct.unpack(MDMFHEADERWITHOUTOFFSETS,
1251 encoding_parameters[:read_size])
1252 if segsize == 0 and datalen == 0:
1253 # Empty file, no segments.
1254 self._num_segments = 0
1256 self._num_segments = mathutil.div_ceil(datalen, segsize)
1258 elif verno == SDMF_VERSION:
1259 read_size = SIGNED_PREFIX_LENGTH
1267 datalen) = struct.unpack(">BQ32s16s BBQQ",
1268 encoding_parameters[:SIGNED_PREFIX_LENGTH])
1270 if segsize == 0 and datalen == 0:
1272 self._num_segments = 0
1274 # non-empty SDMF files have one segment.
1275 self._num_segments = 1
1277 raise UnknownVersionError("You asked me to read mutable file "
1278 "version %d, but I only understand "
1279 "%d and %d" % (verno, SDMF_VERSION,
1282 self._version_number = verno
1283 self._sequence_number = seqnum
1284 self._root_hash = root_hash
1285 self._required_shares = k
1286 self._total_shares = n
1287 self._segment_size = segsize
1288 self._data_length = datalen
1290 self._block_size = self._segment_size / self._required_shares
1291 # We can upload empty files, and need to account for this fact
1292 # so as to avoid zero-division and zero-modulo errors.
1294 tail_size = self._data_length % self._segment_size
1298 self._tail_block_size = self._block_size
1300 self._tail_block_size = mathutil.next_multiple(tail_size,
1301 self._required_shares)
1302 self._tail_block_size /= self._required_shares
1304 return encoding_parameters
1307 def _process_offsets(self, offsets):
1308 if self._version_number == 0:
1309 read_size = OFFSETS_LENGTH
1310 read_offset = SIGNED_PREFIX_LENGTH
1311 end = read_size + read_offset
1317 EOF) = struct.unpack(">LLLLQQ",
1318 offsets[read_offset:end])
1320 self._offsets['signature'] = signature
1321 self._offsets['share_data'] = share_data
1322 self._offsets['block_hash_tree'] = block_hash_tree
1323 self._offsets['share_hash_chain'] = share_hash_chain
1324 self._offsets['enc_privkey'] = enc_privkey
1325 self._offsets['EOF'] = EOF
1327 elif self._version_number == 1:
1328 read_offset = MDMFHEADERWITHOUTOFFSETSSIZE
1329 read_length = MDMFOFFSETS_LENGTH
1330 end = read_offset + read_length
1335 verification_key_end,
1338 eof) = struct.unpack(MDMFOFFSETS,
1339 offsets[read_offset:end])
1341 self._offsets['enc_privkey'] = encprivkey
1342 self._offsets['block_hash_tree'] = blockhashes
1343 self._offsets['share_hash_chain'] = sharehashes
1344 self._offsets['signature'] = signature
1345 self._offsets['verification_key'] = verification_key
1346 self._offsets['verification_key_end']= \
1347 verification_key_end
1348 self._offsets['EOF'] = eof
1349 self._offsets['share_data'] = sharedata
1352 def get_block_and_salt(self, segnum):
1354 I return (block, salt), where block is the block data and
1355 salt is the salt used to encrypt that segment.
1357 d = self._maybe_fetch_offsets_and_header()
1359 base_share_offset = self._offsets['share_data']
1361 if segnum + 1 > self._num_segments:
1362 raise LayoutInvalid("Not a valid segment number")
1364 if self._version_number == 0:
1365 share_offset = base_share_offset + self._block_size * segnum
1367 share_offset = base_share_offset + (self._block_size + \
1369 if segnum + 1 == self._num_segments:
1370 data = self._tail_block_size
1372 data = self._block_size
1374 if self._version_number == 1:
1377 readvs = [(share_offset, data)]
1379 d.addCallback(_then)
1380 d.addCallback(lambda readvs: self._read(readvs))
1381 def _process_results(results):
1382 assert self.shnum in results
1383 if self._version_number == 0:
1384 # We only read the share data, but we know the salt from
1385 # when we fetched the header
1386 data = results[self.shnum]
1390 assert len(data) == 1
1394 data = results[self.shnum]
1398 salt_and_data = results[self.shnum][0]
1399 salt = salt_and_data[:SALT_SIZE]
1400 data = salt_and_data[SALT_SIZE:]
1402 d.addCallback(_process_results)
1406 def get_blockhashes(self, needed=None, force_remote=False):
1408 I return the block hash tree
1410 I take an optional argument, needed, which is a set of indices
1411 correspond to hashes that I should fetch. If this argument is
1412 missing, I will fetch the entire block hash tree; otherwise, I
1413 may attempt to fetch fewer hashes, based on what needed says
1414 that I should do. Note that I may fetch as many hashes as I
1415 want, so long as the set of hashes that I do fetch is a superset
1416 of the ones that I am asked for, so callers should be prepared
1417 to tolerate additional hashes.
1419 # TODO: Return only the parts of the block hash tree necessary
1420 # to validate the blocknum provided?
1421 # This is a good idea, but it is hard to implement correctly. It
1422 # is bad to fetch any one block hash more than once, so we
1423 # probably just want to fetch the whole thing at once and then
1425 if needed == set([]):
1426 return defer.succeed([])
1427 d = self._maybe_fetch_offsets_and_header()
1429 blockhashes_offset = self._offsets['block_hash_tree']
1430 if self._version_number == 1:
1431 blockhashes_length = self._offsets['EOF'] - blockhashes_offset
1433 blockhashes_length = self._offsets['share_data'] - blockhashes_offset
1434 readvs = [(blockhashes_offset, blockhashes_length)]
1436 d.addCallback(_then)
1437 d.addCallback(lambda readvs:
1438 self._read(readvs, force_remote=force_remote))
1439 def _build_block_hash_tree(results):
1440 assert self.shnum in results
1442 rawhashes = results[self.shnum][0]
1443 results = [rawhashes[i:i+HASH_SIZE]
1444 for i in range(0, len(rawhashes), HASH_SIZE)]
1446 d.addCallback(_build_block_hash_tree)
1450 def get_sharehashes(self, needed=None, force_remote=False):
1452 I return the part of the share hash chain placed to validate
1455 I take an optional argument, needed. Needed is a set of indices
1456 that correspond to the hashes that I should fetch. If needed is
1457 not present, I will fetch and return the entire share hash
1458 chain. Otherwise, I may fetch and return any part of the share
1459 hash chain that is a superset of the part that I am asked to
1460 fetch. Callers should be prepared to deal with more hashes than
1463 if needed == set([]):
1464 return defer.succeed([])
1465 d = self._maybe_fetch_offsets_and_header()
1467 def _make_readvs(ignored):
1468 sharehashes_offset = self._offsets['share_hash_chain']
1469 if self._version_number == 0:
1470 sharehashes_length = self._offsets['block_hash_tree'] - sharehashes_offset
1472 sharehashes_length = self._offsets['signature'] - sharehashes_offset
1473 readvs = [(sharehashes_offset, sharehashes_length)]
1475 d.addCallback(_make_readvs)
1476 d.addCallback(lambda readvs:
1477 self._read(readvs, force_remote=force_remote))
1478 def _build_share_hash_chain(results):
1479 assert self.shnum in results
1481 sharehashes = results[self.shnum][0]
1482 results = [sharehashes[i:i+(HASH_SIZE + 2)]
1483 for i in range(0, len(sharehashes), HASH_SIZE + 2)]
1484 results = dict([struct.unpack(">H32s", data)
1485 for data in results])
1487 d.addCallback(_build_share_hash_chain)
1491 def get_encprivkey(self):
1493 I return the encrypted private key.
1495 d = self._maybe_fetch_offsets_and_header()
1497 def _make_readvs(ignored):
1498 privkey_offset = self._offsets['enc_privkey']
1499 if self._version_number == 0:
1500 privkey_length = self._offsets['EOF'] - privkey_offset
1502 privkey_length = self._offsets['share_hash_chain'] - privkey_offset
1503 readvs = [(privkey_offset, privkey_length)]
1505 d.addCallback(_make_readvs)
1506 d.addCallback(lambda readvs: self._read(readvs))
1507 def _process_results(results):
1508 assert self.shnum in results
1509 privkey = results[self.shnum][0]
1511 d.addCallback(_process_results)
1515 def get_signature(self):
1517 I return the signature of my share.
1519 d = self._maybe_fetch_offsets_and_header()
1521 def _make_readvs(ignored):
1522 signature_offset = self._offsets['signature']
1523 if self._version_number == 1:
1524 signature_length = self._offsets['verification_key'] - signature_offset
1526 signature_length = self._offsets['share_hash_chain'] - signature_offset
1527 readvs = [(signature_offset, signature_length)]
1529 d.addCallback(_make_readvs)
1530 d.addCallback(lambda readvs: self._read(readvs))
1531 def _process_results(results):
1532 assert self.shnum in results
1533 signature = results[self.shnum][0]
1535 d.addCallback(_process_results)
1539 def get_verification_key(self):
1541 I return the verification key.
1543 d = self._maybe_fetch_offsets_and_header()
1545 def _make_readvs(ignored):
1546 if self._version_number == 1:
1547 vk_offset = self._offsets['verification_key']
1548 vk_length = self._offsets['verification_key_end'] - vk_offset
1550 vk_offset = struct.calcsize(">BQ32s16sBBQQLLLLQQ")
1551 vk_length = self._offsets['signature'] - vk_offset
1552 readvs = [(vk_offset, vk_length)]
1554 d.addCallback(_make_readvs)
1555 d.addCallback(lambda readvs: self._read(readvs))
1556 def _process_results(results):
1557 assert self.shnum in results
1558 verification_key = results[self.shnum][0]
1559 return verification_key
1560 d.addCallback(_process_results)
1564 def get_encoding_parameters(self):
1566 I return (k, n, segsize, datalen)
1568 d = self._maybe_fetch_offsets_and_header()
1569 d.addCallback(lambda ignored:
1570 (self._required_shares,
1577 def get_seqnum(self):
1579 I return the sequence number for this share.
1581 d = self._maybe_fetch_offsets_and_header()
1582 d.addCallback(lambda ignored:
1583 self._sequence_number)
1587 def get_root_hash(self):
1589 I return the root of the block hash tree
1591 d = self._maybe_fetch_offsets_and_header()
1592 d.addCallback(lambda ignored: self._root_hash)
1596 def get_checkstring(self):
1598 I return the packed representation of the following:
1605 which my users use as a checkstring to detect other writers.
1607 d = self._maybe_fetch_offsets_and_header()
1608 def _build_checkstring(ignored):
1610 checkstring = struct.pack(PREFIX,
1611 self._version_number,
1612 self._sequence_number,
1616 checkstring = struct.pack(MDMFCHECKSTRING,
1617 self._version_number,
1618 self._sequence_number,
1622 d.addCallback(_build_checkstring)
1626 def get_prefix(self, force_remote):
1627 d = self._maybe_fetch_offsets_and_header(force_remote)
1628 d.addCallback(lambda ignored:
1629 self._build_prefix())
1633 def _build_prefix(self):
1634 # The prefix is another name for the part of the remote share
1635 # that gets signed. It consists of everything up to and
1636 # including the datalength, packed by struct.
1637 if self._version_number == SDMF_VERSION:
1638 return struct.pack(SIGNED_PREFIX,
1639 self._version_number,
1640 self._sequence_number,
1643 self._required_shares,
1649 return struct.pack(MDMFSIGNABLEHEADER,
1650 self._version_number,
1651 self._sequence_number,
1653 self._required_shares,
1659 def _get_offsets_tuple(self):
1660 # The offsets tuple is another component of the version
1661 # information tuple. It is basically our offsets dictionary,
1662 # itemized and in a tuple.
1663 return self._offsets.copy()
1666 def get_verinfo(self):
1668 I return my verinfo tuple. This is used by the ServermapUpdater
1669 to keep track of versions of mutable files.
1671 The verinfo tuple for MDMF files contains:
1679 - prefix (the thing that you sign)
1680 - a tuple of offsets
1682 We include the nonce in MDMF to simplify processing of version
1685 The verinfo tuple for SDMF files is the same, but contains a
1686 16-byte IV instead of a hash of salts.
1688 d = self._maybe_fetch_offsets_and_header()
1689 def _build_verinfo(ignored):
1690 if self._version_number == SDMF_VERSION:
1691 salt_to_use = self._salt
1694 return (self._sequence_number,
1699 self._required_shares,
1701 self._build_prefix(),
1702 self._get_offsets_tuple())
1703 d.addCallback(_build_verinfo)
1707 def _read(self, readvs, force_remote=False):
1708 unsatisfiable = filter(lambda x: x[0] + x[1] > len(self._data), readvs)
1709 # TODO: It's entirely possible to tweak this so that it just
1710 # fulfills the requests that it can, and not demand that all
1711 # requests are satisfiable before running it.
1712 if not unsatisfiable and not force_remote:
1713 results = [self._data[offset:offset+length]
1714 for (offset, length) in readvs]
1715 results = {self.shnum: results}
1716 return defer.succeed(results)
1718 return self._rref.callRemote("slot_readv",
1719 self._storage_index,
1725 """I tell my caller whether or not my remote file is SDMF or MDMF
1727 d = self._maybe_fetch_offsets_and_header()
1728 d.addCallback(lambda ignored:
1729 self._version_number == 0)
1733 class LayoutInvalid(Exception):
1735 This isn't a valid MDMF mutable file