2 from zope.interface import Interface
3 from foolscap.schema import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
5 from foolscap import RemoteInterface, Referenceable
9 Hash = StringConstraint(maxLength=HASH_SIZE,
10 minLength=HASH_SIZE)# binary format 32-byte SHA256 hash
11 Nodeid = StringConstraint(maxLength=20,
12 minLength=20) # binary format 20-byte SHA1 hash
13 FURL = StringConstraint(1000)
14 StorageIndex = StringConstraint(16)
15 URI = StringConstraint(300) # kind of arbitrary
16 MAX_BUCKETS = 200 # per peer
17 ShareData = StringConstraint(400000) # 1MB segment / k=3 = 334kB
18 URIExtensionData = StringConstraint(1000)
19 LeaseRenewSecret = Hash # used to protect bucket lease renewal requests
20 LeaseCancelSecret = Hash # used to protect bucket lease cancellation requests
23 class RIIntroducerClient(RemoteInterface):
24 def new_peers(furls=SetOf(FURL)):
26 def set_encoding_parameters(parameters=(int, int, int)):
27 """Advise the client of the recommended k-of-n encoding parameters
28 for this grid. 'parameters' is a tuple of (k, desired, n), where 'n'
29 is the total number of shares that will be created for any given
30 file, while 'k' is the number of shares that must be retrieved to
31 recover that file, and 'desired' is the minimum number of shares that
32 must be placed before the uploader will consider its job a success.
33 n/k is the expansion ratio, while k determines the robustness.
35 Introducers should specify 'n' according to the expected size of the
36 grid (there is no point to producing more shares than there are
37 peers), and k according to the desired reliability-vs-overhead goals.
39 Note that setting k=1 is equivalent to simple replication.
43 class RIIntroducer(RemoteInterface):
44 def hello(node=RIIntroducerClient, furl=FURL):
47 class RIClient(RemoteInterface):
49 """Return a tuple of (my_version, oldest_supported) strings.
51 Each string can be parsed by an allmydata.util.version.Version
52 instance, and then compared. The first goal is to make sure that
53 nodes are not confused by speaking to an incompatible peer. The
54 second goal is to enable the development of backwards-compatibility
57 This method is likely to change in incompatible ways until we get the
58 whole compatibility scheme nailed down.
60 return TupleOf(str, str)
61 def get_service(name=str):
66 class RIBucketWriter(RemoteInterface):
67 def write(offset=int, data=ShareData):
72 If the data that has been written is incomplete or inconsistent then
73 the server will throw the data away, else it will store it for future
78 class RIBucketReader(RemoteInterface):
79 def read(offset=int, length=int):
83 class RIStorageServer(RemoteInterface):
84 def allocate_buckets(storage_index=StorageIndex,
85 renew_secret=LeaseRenewSecret,
86 cancel_secret=LeaseCancelSecret,
87 sharenums=SetOf(int, maxLength=MAX_BUCKETS),
88 allocated_size=int, canary=Referenceable):
90 @param storage_index: the index of the bucket to be created or
92 @param sharenums: these are the share numbers (probably between 0 and
93 99) that the sender is proposing to store on this
95 @param renew_secret: This is the secret used to protect bucket refresh
96 This secret is generated by the client and
97 stored for later comparison by the server. Each
98 server is given a different secret.
99 @param cancel_secret: Like renew_secret, but protects bucket decref.
100 @param canary: If the canary is lost before close(), the bucket is
102 @return: tuple of (alreadygot, allocated), where alreadygot is what we
103 already have and is what we hereby agree to accept. New
104 leases are added for shares in both lists.
106 return TupleOf(SetOf(int, maxLength=MAX_BUCKETS),
107 DictOf(int, RIBucketWriter, maxKeys=MAX_BUCKETS))
108 def get_buckets(storage_index=StorageIndex):
109 return DictOf(int, RIBucketReader, maxKeys=MAX_BUCKETS)
112 class IStorageBucketWriter(Interface):
113 def put_block(segmentnum=int, data=ShareData):
114 """@param data: For most segments, this data will be 'blocksize'
115 bytes in length. The last segment might be shorter.
116 @return: a Deferred that fires (with None) when the operation completes
119 def put_plaintext_hashes(hashes=ListOf(Hash, maxLength=2**20)):
121 @return: a Deferred that fires (with None) when the operation completes
124 def put_crypttext_hashes(hashes=ListOf(Hash, maxLength=2**20)):
126 @return: a Deferred that fires (with None) when the operation completes
129 def put_block_hashes(blockhashes=ListOf(Hash, maxLength=2**20)):
131 @return: a Deferred that fires (with None) when the operation completes
134 def put_share_hashes(sharehashes=ListOf(TupleOf(int, Hash),
137 @return: a Deferred that fires (with None) when the operation completes
140 def put_uri_extension(data=URIExtensionData):
141 """This block of data contains integrity-checking information (hashes
142 of plaintext, crypttext, and shares), as well as encoding parameters
143 that are necessary to recover the data. This is a serialized dict
144 mapping strings to other strings. The hash of this data is kept in
145 the URI and verified before any of the data is used. All buckets for
146 a given file contain identical copies of this data.
148 The serialization format is specified with the following pseudocode:
149 for k in sorted(dict.keys()):
150 assert re.match(r'^[a-zA-Z_\-]+$', k)
151 write(k + ':' + netstring(dict[k]))
153 @return: a Deferred that fires (with None) when the operation completes
157 """Finish writing and close the bucket. The share is not finalized
158 until this method is called: if the uploading client disconnects
159 before calling close(), the partially-written share will be
162 @return: a Deferred that fires (with None) when the operation completes
165 class IStorageBucketReader(Interface):
167 def get_block(blocknum=int):
168 """Most blocks will be the same size. The last block might be shorter
174 def get_plaintext_hashes():
176 @return: ListOf(Hash, maxLength=2**20)
179 def get_crypttext_hashes():
181 @return: ListOf(Hash, maxLength=2**20)
184 def get_block_hashes():
186 @return: ListOf(Hash, maxLength=2**20)
189 def get_share_hashes():
191 @return: ListOf(TupleOf(int, Hash), maxLength=2**20)
194 def get_uri_extension():
196 @return: URIExtensionData
201 # hm, we need a solution for forward references in schemas
202 from foolscap.schema import Any
203 RIMutableDirectoryNode_ = Any() # TODO: how can we avoid this?
205 FileNode_ = Any() # TODO: foolscap needs constraints on copyables
206 DirectoryNode_ = Any() # TODO: same
207 AnyNode_ = ChoiceOf(FileNode_, DirectoryNode_)
210 class RIVirtualDriveServer(RemoteInterface):
211 def get_public_root_uri():
212 """Obtain the URI for this server's global publically-writable root
213 directory. This returns a read-write directory URI.
215 If this vdrive server does not offer a public root, this will
216 raise an exception."""
219 def create_directory(index=Hash, write_enabler=Hash):
220 """Create a new (empty) directory, unattached to anything else.
222 This returns the same index that was passed in.
226 def get(index=Hash, key=Hash):
227 """Retrieve a named child of the given directory. 'index' specifies
228 which directory is being accessed, and is generally the hash of the
229 read key. 'key' is the hash of the read key and the child name.
231 This operation returns a pair of encrypted strings. The first string
232 is meant to be decrypted by the Write Key and provides read-write
233 access to the child. If this directory holds read-only access to the
234 child, this first string will be an empty string. The second string
235 is meant to be decrypted by the Read Key and provides read-only
238 When the child is a read-write directory, the encrypted URI:DIR-RO
239 will be in the read slot, and the encrypted URI:DIR will be in the
240 write slot. When the child is a read-only directory, the encrypted
241 URI:DIR-RO will be in the read slot and the write slot will be empty.
242 When the child is a CHK file, the encrypted URI:CHK will be in the
243 read slot, and the write slot will be empty.
245 This might raise IndexError if there is no child by the desired name.
247 return (EncryptedThing, EncryptedThing)
249 def list(index=Hash):
250 """List the contents of a directory.
252 This returns a list of (NAME, WRITE, READ) tuples. Each value is an
253 encrypted string (although the WRITE value may sometimes be an empty
256 NAME: the child name, encrypted with the Read Key
257 WRITE: the child write URI, encrypted with the Write Key, or an
258 empty string if this child is read-only
259 READ: the child read URI, encrypted with the Read Key
261 return ListOf((EncryptedThing, EncryptedThing, EncryptedThing),
265 def set(index=Hash, write_enabler=Hash, key=Hash,
266 name=EncryptedThing, write=EncryptedThing, read=EncryptedThing):
267 """Set a child object. I will replace any existing child of the same
271 def delete(index=Hash, write_enabler=Hash, key=Hash):
272 """Delete a specific child.
274 This uses the hashed key to locate a specific child, and deletes it.
278 class IURI(Interface):
279 def init_from_string(uri):
280 """Accept a string (as created by my to_string() method) and populate
281 this instance with its data. I am not normally called directly,
282 please use the module-level uri.from_string() function to convert
283 arbitrary URI strings into IURI-providing instances."""
286 """Return False if this URI be used to modify the data. Return True
287 if this URI cannot be used to modify the data."""
290 """Return True if the data can be modified by *somebody* (perhaps
291 someone who has a more powerful URI than this one)."""
294 """Return another IURI instance, which represents a read-only form of
295 this one. If is_readonly() is True, this returns self."""
298 """Return a string of printable ASCII characters, suitable for
299 passing into init_from_string."""
301 class IDirnodeURI(Interface):
302 """I am a URI which represents a dirnode."""
304 class IFileURI(Interface):
305 """I am a URI which represents a filenode."""
307 """Return the length (in bytes) of the file that I represent."""
310 class IFileNode(Interface):
311 def download(target):
312 """Download the file's contents to a given IDownloadTarget"""
313 def download_to_data():
314 """Download the file's contents. Return a Deferred that fires
315 with those contents."""
318 """Return the URI that can be used by others to get access to this
322 """Return the length (in bytes) of the data this node represents."""
324 def get_refresh_capability():
325 """Return a string that represents the 'refresh capability' for this
326 node. The holder of this capability will be able to renew the lease
327 for this node, protecting it from garbage-collection.
330 class IDirectoryNode(Interface):
332 """Return True if this directory is mutable, False if it is read-only.
336 """Return the directory URI that can be used by others to get access
337 to this directory node. If this node is read-only, the URI will only
338 offer read-only access. If this node is read-write, the URI will
339 offer read-write acess.
341 If you have read-write access to a directory and wish to share merely
342 read-only access with others, use get_immutable_uri().
344 The dirnode ('1') URI returned by this method can be used in
345 set_uri() on a different directory ('2') to 'mount' a reference to
346 this directory ('1') under the other ('2'). This URI is just a
347 string, so it can be passed around through email or other out-of-band
351 def get_immutable_uri():
352 """Return the directory URI that can be used by others to get
353 read-only access to this directory node. The result is a read-only
354 URI, regardless of whether this dirnode is read-only or read-write.
356 If you have merely read-only access to this dirnode,
357 get_immutable_uri() will return the same thing as get_uri().
360 def get_refresh_capability():
361 """Return a string that represents the 'refresh capability' for this
362 node. The holder of this capability will be able to renew the lease
363 for this node, protecting it from garbage-collection.
367 """I return a Deferred that fires with a dictionary mapping child
368 name to an IFileNode or IDirectoryNode."""
371 """I return a Deferred that fires with a boolean, True if there
372 exists a child of the given name, False if not."""
375 """I return a Deferred that fires with a specific named child node,
376 either an IFileNode or an IDirectoryNode."""
378 def get_child_at_path(path):
379 """Transform a child path into an IDirectoryNode or IFileNode.
381 I perform a recursive series of 'get' operations to find the named
382 descendant node. I return a Deferred that fires with the node, or
383 errbacks with IndexError if the node could not be found.
385 The path can be either a single string (slash-separated) or a list of
389 def set_uri(name, child_uri):
390 """I add a child (by URI) at the specific name. I return a Deferred
391 that fires when the operation finishes. I will replace any existing
392 child of the same name.
394 The child_uri could be for a file, or for a directory (either
395 read-write or read-only, using a URI that came from get_uri() ).
397 If this directory node is read-only, the Deferred will errback with a
400 def set_node(name, child):
401 """I add a child at the specific name. I return a Deferred that fires
402 when the operation finishes. This Deferred will fire with the child
403 node that was just added. I will replace any existing child of the
406 If this directory node is read-only, the Deferred will errback with a
409 def add_file(name, uploadable):
410 """I upload a file (using the given IUploadable), then attach the
411 resulting FileNode to the directory at the given name. I return a
412 Deferred that fires (with the IFileNode of the uploaded file) when
413 the operation completes."""
416 """I remove the child at the specific name. I return a Deferred that
417 fires when the operation finishes."""
419 def create_empty_directory(name):
420 """I create and attach an empty directory at the given name. I return
421 a Deferred that fires when the operation finishes."""
423 def move_child_to(current_child_name, new_parent, new_child_name=None):
424 """I take one of my children and move them to a new parent. The child
425 is referenced by name. On the new parent, the child will live under
426 'new_child_name', which defaults to 'current_child_name'. I return a
427 Deferred that fires when the operation finishes."""
429 def build_manifest():
430 """Return a set of refresh-capabilities for all nodes (directories
431 and files) reachable from this one."""
433 class ICodecEncoder(Interface):
434 def set_params(data_size, required_shares, max_shares):
435 """Set up the parameters of this encoder.
437 This prepares the encoder to perform an operation that converts a
438 single block of data into a number of shares, such that a future
439 ICodecDecoder can use a subset of these shares to recover the
440 original data. This operation is invoked by calling encode(). Once
441 the encoding parameters are set up, the encode operation can be
442 invoked multiple times.
444 set_params() prepares the encoder to accept blocks of input data that
445 are exactly 'data_size' bytes in length. The encoder will be prepared
446 to produce 'max_shares' shares for each encode() operation (although
447 see the 'desired_share_ids' to use less CPU). The encoding math will
448 be chosen such that the decoder can get by with as few as
449 'required_shares' of these shares and still reproduce the original
450 data. For example, set_params(1000, 5, 5) offers no redundancy at
451 all, whereas set_params(1000, 1, 10) provides 10x redundancy.
453 Numerical Restrictions: 'data_size' is required to be an integral
454 multiple of 'required_shares'. In general, the caller should choose
455 required_shares and max_shares based upon their reliability
456 requirements and the number of peers available (the total storage
457 space used is roughly equal to max_shares*data_size/required_shares),
458 then choose data_size to achieve the memory footprint desired (larger
459 data_size means more efficient operation, smaller data_size means
460 smaller memory footprint).
462 In addition, 'max_shares' must be equal to or greater than
463 'required_shares'. Of course, setting them to be equal causes
464 encode() to degenerate into a particularly slow form of the 'split'
467 See encode() for more details about how these parameters are used.
469 set_params() must be called before any other ICodecEncoder methods
473 def get_encoder_type():
474 """Return a short string that describes the type of this encoder.
476 There is required to be a global table of encoder classes. This method
477 returns an index into this table; the value at this index is an
478 encoder class, and this encoder is an instance of that class.
481 def get_serialized_params(): # TODO: maybe, maybe not
482 """Return a string that describes the parameters of this encoder.
484 This string can be passed to the decoder to prepare it for handling
485 the encoded shares we create. It might contain more information than
486 was presented to set_params(), if there is some flexibility of
489 This string is intended to be embedded in the URI, so there are
490 several restrictions on its contents. At the moment I'm thinking that
491 this means it may contain hex digits and hyphens, and nothing else.
492 The idea is that the URI contains something like '%s:%s:%s' %
493 (encoder.get_encoder_name(), encoder.get_serialized_params(),
494 b2a(crypttext_hash)), and this is enough information to construct a
498 def get_block_size():
499 """Return the length of the shares that encode() will produce.
502 def encode_proposal(data, desired_share_ids=None):
505 'data' must be a string (or other buffer object), and len(data) must
506 be equal to the 'data_size' value passed earlier to set_params().
508 This will return a Deferred that will fire with two lists. The first
509 is a list of shares, each of which is a string (or other buffer
510 object) such that len(share) is the same as what get_share_size()
511 returned earlier. The second is a list of shareids, in which each is
512 an integer. The lengths of the two lists will always be equal to each
513 other. The user should take care to keep each share closely
514 associated with its shareid, as one is useless without the other.
516 The length of this output list will normally be the same as the value
517 provided to the 'max_shares' parameter of set_params(). This may be
518 different if 'desired_share_ids' is provided.
520 'desired_share_ids', if provided, is required to be a sequence of
521 ints, each of which is required to be >= 0 and < max_shares. If not
522 provided, encode() will produce 'max_shares' shares, as if
523 'desired_share_ids' were set to range(max_shares). You might use this
524 if you initially thought you were going to use 10 peers, started
525 encoding, and then two of the peers dropped out: you could use
526 desired_share_ids= to skip the work (both memory and CPU) of
527 producing shares for the peers which are no longer available.
531 def encode(inshares, desired_share_ids=None):
532 """Encode some data. This may be called multiple times. Each call is
535 inshares is a sequence of length required_shares, containing buffers
536 (i.e. strings), where each buffer contains the next contiguous
537 non-overlapping segment of the input data. Each buffer is required to
538 be the same length, and the sum of the lengths of the buffers is
539 required to be exactly the data_size promised by set_params(). (This
540 implies that the data has to be padded before being passed to
541 encode(), unless of course it already happens to be an even multiple
542 of required_shares in length.)
544 ALSO: the requirement to break up your data into 'required_shares'
545 chunks before calling encode() feels a bit surprising, at least from
546 the point of view of a user who doesn't know how FEC works. It feels
547 like an implementation detail that has leaked outside the
548 abstraction barrier. Can you imagine a use case in which the data to
549 be encoded might already be available in pre-segmented chunks, such
550 that it is faster or less work to make encode() take a list rather
551 than splitting a single string?
553 ALSO ALSO: I think 'inshares' is a misleading term, since encode()
554 is supposed to *produce* shares, so what it *accepts* should be
555 something other than shares. Other places in this interface use the
556 word 'data' for that-which-is-not-shares.. maybe we should use that
559 'desired_share_ids', if provided, is required to be a sequence of
560 ints, each of which is required to be >= 0 and < max_shares. If not
561 provided, encode() will produce 'max_shares' shares, as if
562 'desired_share_ids' were set to range(max_shares). You might use this
563 if you initially thought you were going to use 10 peers, started
564 encoding, and then two of the peers dropped out: you could use
565 desired_share_ids= to skip the work (both memory and CPU) of
566 producing shares for the peers which are no longer available.
568 For each call, encode() will return a Deferred that fires with two
569 lists, one containing shares and the other containing the shareids.
570 The get_share_size() method can be used to determine the length of
571 the share strings returned by encode(). Each shareid is a small
572 integer, exactly as passed into 'desired_share_ids' (or
573 range(max_shares), if desired_share_ids was not provided).
575 The shares and their corresponding shareids are required to be kept
576 together during storage and retrieval. Specifically, the share data is
577 useless by itself: the decoder needs to be told which share is which
578 by providing it with both the shareid and the actual share data.
580 This function will allocate an amount of memory roughly equal to::
582 (max_shares - required_shares) * get_share_size()
584 When combined with the memory that the caller must allocate to
585 provide the input data, this leads to a memory footprint roughly
586 equal to the size of the resulting encoded shares (i.e. the expansion
587 factor times the size of the input segment).
592 # returning a list of (shareidN,shareN) tuples instead of a pair of
593 # lists (shareids..,shares..). Brian thought the tuples would
594 # encourage users to keep the share and shareid together throughout
595 # later processing, Zooko pointed out that the code to iterate
596 # through two lists is not really more complicated than using a list
597 # of tuples and there's also a performance improvement
599 # having 'data_size' not required to be an integral multiple of
600 # 'required_shares'. Doing this would require encode() to perform
601 # padding internally, and we'd prefer to have any padding be done
602 # explicitly by the caller. Yes, it is an abstraction leak, but
603 # hopefully not an onerous one.
606 class ICodecDecoder(Interface):
607 def set_serialized_params(params):
608 """Set up the parameters of this encoder, from a string returned by
609 encoder.get_serialized_params()."""
611 def get_needed_shares():
612 """Return the number of shares needed to reconstruct the data.
613 set_serialized_params() is required to be called before this."""
615 def decode(some_shares, their_shareids):
616 """Decode a partial list of shares into data.
618 'some_shares' is required to be a sequence of buffers of sharedata, a
619 subset of the shares returned by ICodecEncode.encode(). Each share is
620 required to be of the same length. The i'th element of their_shareids
621 is required to be the shareid of the i'th buffer in some_shares.
623 This returns a Deferred which fires with a sequence of buffers. This
624 sequence will contain all of the segments of the original data, in
625 order. The sum of the lengths of all of the buffers will be the
626 'data_size' value passed into the original ICodecEncode.set_params()
627 call. To get back the single original input block of data, use
628 ''.join(output_buffers), or you may wish to simply write them in
629 order to an output file.
631 Note that some of the elements in the result sequence may be
632 references to the elements of the some_shares input sequence. In
633 particular, this means that if those share objects are mutable (e.g.
634 arrays) and if they are changed, then both the input (the
635 'some_shares' parameter) and the output (the value given when the
636 deferred is triggered) will change.
638 The length of 'some_shares' is required to be exactly the value of
639 'required_shares' passed into the original ICodecEncode.set_params()
643 class IEncoder(Interface):
644 """I take an object that provides IEncryptedUploadable, which provides
645 encrypted data, and a list of shareholders. I then encode, hash, and
646 deliver shares to those shareholders. I will compute all the necessary
647 Merkle hash trees that are necessary to validate the crypttext that
648 eventually comes back from the shareholders. I provide the URI Extension
649 Block Hash, and the encoding parameters, both of which must be included
652 I do not choose shareholders, that is left to the IUploader. I must be
653 given a dict of RemoteReferences to storage buckets that are ready and
654 willing to receive data.
658 """Specify the number of bytes that will be encoded. This must be
659 peformed before get_serialized_params() can be called.
661 def set_params(params):
662 """Override the default encoding parameters. 'params' is a tuple of
663 (k,d,n), where 'k' is the number of required shares, 'd' is the
664 shares_of_happiness, and 'n' is the total number of shares that will
667 Encoding parameters can be set in three ways. 1: The Encoder class
668 provides defaults (25/75/100). 2: the Encoder can be constructed with
669 an 'options' dictionary, in which the
670 needed_and_happy_and_total_shares' key can be a (k,d,n) tuple. 3:
671 set_params((k,d,n)) can be called.
673 If you intend to use set_params(), you must call it before
674 get_share_size or get_param are called.
677 def set_encrypted_uploadable(u):
678 """Provide a source of encrypted upload data. 'u' must implement
679 IEncryptedUploadable.
681 When this is called, the IEncryptedUploadable will be queried for its
682 length and the storage_index that should be used.
684 This returns a Deferred that fires with this Encoder instance.
686 This must be performed before start() can be called.
690 """Return an encoding parameter, by name.
692 'storage_index': return a string with the (16-byte truncated SHA-256
693 hash) storage index to which these shares should be
696 'share_counts': return a tuple describing how many shares are used:
697 (needed_shares, shares_of_happiness, total_shares)
699 'num_segments': return an int with the number of segments that
702 'segment_size': return an int with the size of each segment.
704 'block_size': return the size of the individual blocks that will
705 be delivered to a shareholder's put_block() method. By
706 knowing this, the shareholder will be able to keep all
707 blocks in a single file and still provide random access
708 when reading them. # TODO: can we avoid exposing this?
710 'share_size': an int with the size of the data that will be stored
711 on each shareholder. This is aggregate amount of data
712 that will be sent to the shareholder, summed over all
713 the put_block() calls I will ever make. It is useful to
714 determine this size before asking potential
715 shareholders whether they will grant a lease or not,
716 since their answers will depend upon how much space we
717 need. TODO: this might also include some amount of
718 overhead, like the size of all the hashes. We need to
719 decide whether this is useful or not.
721 'serialized_params': a string with a concise description of the
722 codec name and its parameters. This may be passed
723 into the IUploadable to let it make sure that
724 the same file encoded with different parameters
725 will result in different storage indexes.
727 Once this is called, set_size() and set_params() may not be called.
730 def set_shareholders(shareholders):
731 """Tell the encoder where to put the encoded shares. 'shareholders'
732 must be a dictionary that maps share number (an integer ranging from
733 0 to n-1) to an instance that provides IStorageBucketWriter. This
734 must be performed before start() can be called."""
737 """Begin the encode/upload process. This involves reading encrypted
738 data from the IEncryptedUploadable, encoding it, uploading the shares
739 to the shareholders, then sending the hash trees.
741 set_encrypted_uploadable() and set_shareholders() must be called
742 before this can be invoked.
744 This returns a Deferred that fires with a tuple of
745 (uri_extension_hash, needed_shares, total_shares, size) when the
746 upload process is complete. This information, plus the encryption
747 key, is sufficient to construct the URI.
750 class IDecoder(Interface):
751 """I take a list of shareholders and some setup information, then
752 download, validate, decode, and decrypt data from them, writing the
753 results to an output file.
755 I do not locate the shareholders, that is left to the IDownloader. I must
756 be given a dict of RemoteReferences to storage buckets that are ready to
761 """I take a file-like object (providing write and close) to which all
762 the plaintext data will be written.
764 TODO: producer/consumer . Maybe write() should return a Deferred that
765 indicates when it will accept more data? But probably having the
766 IDecoder be a producer is easier to glue to IConsumer pieces.
769 def set_shareholders(shareholders):
770 """I take a dictionary that maps share identifiers (small integers)
771 to RemoteReferences that provide RIBucketReader. This must be called
775 """I start the download. This process involves retrieving data and
776 hash chains from the shareholders, using the hashes to validate the
777 data, decoding the shares into segments, decrypting the segments,
778 then writing the resulting plaintext to the output file.
780 I return a Deferred that will fire (with self) when the download is
784 class IDownloadTarget(Interface):
786 """Called before any calls to write() or close(). If an error
787 occurs before any data is available, fail() may be called without
788 a previous call to open().
790 'size' is the length of the file being downloaded, in bytes."""
793 """Output some data to the target."""
795 """Inform the target that there is no more data to be written."""
797 """fail() is called to indicate that the download has failed. 'why'
798 is a Failure object indicating what went wrong. No further methods
799 will be invoked on the IDownloadTarget after fail()."""
800 def register_canceller(cb):
801 """The FileDownloader uses this to register a no-argument function
802 that the target can call to cancel the download. Once this canceller
803 is invoked, no further calls to write() or close() will be made."""
805 """When the FileDownloader is done, this finish() function will be
806 called. Whatever it returns will be returned to the invoker of
810 class IDownloader(Interface):
811 def download(uri, target):
812 """Perform a CHK download, sending the data to the given target.
813 'target' must provide IDownloadTarget.
815 Returns a Deferred that fires (with the results of target.finish)
816 when the download is finished, or errbacks if something went wrong."""
818 class IEncryptedUploadable(Interface):
820 """This behaves just like IUploadable.get_size()."""
822 def set_serialized_encoding_parameters(serialized_encoding_parameters):
823 """Tell me what encoding parameters will be used for my data.
825 'serialized_encoding_parameters' is a string which indicates how the
826 data will be encoded (codec name, blocksize, number of shares).
828 I may use this when get_storage_index() is called, to influence the
829 index that I return. Or, I may just ignore it.
831 set_serialized_encoding_parameters() may be called 0 or 1 times. If
832 called, it must be called before get_storage_index().
835 def get_storage_index():
836 """Return a Deferred that fires with a 16-byte storage index. This
837 value may be influenced by the parameters earlier set by
838 set_serialized_encoding_parameters().
841 def set_segment_size(segment_size):
842 """Set the segment size, to allow the IEncryptedUploadable to
843 accurately create the plaintext segment hash tree. This must be
844 called before any calls to read_encrypted."""
846 def read_encrypted(length):
847 """This behaves just like IUploadable.read(), but returns crypttext
848 instead of plaintext. set_segment_size() must be called before the
849 first call to read_encrypted()."""
851 def get_plaintext_segment_hashtree_nodes(num_segments):
852 """Get the nodes of a merkle hash tree over the plaintext segments.
854 This returns a Deferred which fires with a sequence of hashes. Each
855 hash is a node of a merkle hash tree, generally obtained from::
857 tuple(HashTree(segment_hashes))
859 'num_segments' is used to assert that the number of segments that the
860 IEncryptedUploadable handled matches the number of segments that the
861 encoder was expecting.
864 def get_plaintext_hash():
865 """Get the hash of the whole plaintext.
867 This returns a Deferred which fires with a tagged SHA-256 hash of the
868 whole plaintext, obtained from hashutil.plaintext_hash(data).
872 """Just like IUploadable.close()."""
874 class IUploadable(Interface):
876 """Return a Deferred that will fire with the length of the data to be
877 uploaded, in bytes. This will be called before the data is actually
878 used, to compute encoding parameters.
881 def set_serialized_encoding_parameters(serialized_encoding_parameters):
882 """Tell me what encoding parameters will be used for my data.
884 'serialized_encoding_parameters' is a string which indicates how the
885 data will be encoded (codec name, blocksize, number of shares).
887 I may use this when get_encryption_key() is called, to influence the
888 key that I return. Or, I may just ignore it.
890 set_serialized_encoding_parameters() may be called 0 or 1 times. If
891 called, it must be called before get_encryption_key().
894 def get_encryption_key():
895 """Return a Deferred that fires with a 16-byte AES key. This key will
896 be used to encrypt the data. The key will also be hashed to derive
899 Uploadables which want to achieve convergence should hash their file
900 contents and the serialized_encoding_parameters to form the key
901 (which of course requires a full pass over the data). Uploadables can
902 use the upload.ConvergentUploadMixin class to achieve this
905 Uploadables which do not care about convergence (or do not wish to
906 make multiple passes over the data) can simply return a
907 strongly-random 16 byte string.
909 get_encryption_key() may be called multiple times: the IUploadable is
910 required to return the same value each time.
914 """Return a Deferred that fires with a list of strings (perhaps with
915 only a single element) which, when concatenated together, contain the
916 next 'length' bytes of data. If EOF is near, this may provide fewer
917 than 'length' bytes. The total number of bytes provided by read()
918 before it signals EOF must equal the size provided by get_size().
920 If the data must be acquired through multiple internal read
921 operations, returning a list instead of a single string may help to
922 reduce string copies.
924 'length' will typically be equal to (min(get_size(),1MB)/req_shares),
925 so a 10kB file means length=3kB, 100kB file means length=30kB,
926 and >=1MB file means length=300kB.
928 This method provides for a single full pass through the data. Later
929 use cases may desire multiple passes or access to only parts of the
930 data (such as a mutable file making small edits-in-place). This API
931 will be expanded once those use cases are better understood.
935 """The upload is finished, and whatever filehandle was in use may be
938 class IUploader(Interface):
939 def upload(uploadable):
940 """Upload the file. 'uploadable' must impement IUploadable. This
941 returns a Deferred which fires with the URI of the file."""
943 def upload_ssk(write_capability, new_version, uploadable):
944 """TODO: how should this work?"""
945 def upload_data(data):
946 """Like upload(), but accepts a string."""
948 def upload_filename(filename):
949 """Like upload(), but accepts an absolute pathname."""
951 def upload_filehandle(filehane):
952 """Like upload(), but accepts an open filehandle."""
954 class IVirtualDrive(Interface):
955 """I am a service that may be available to a client.
957 Within any client program, this service can be retrieved by using
958 client.getService('vdrive').
961 def have_public_root():
962 """Return a Boolean, True if get_public_root() will work."""
963 def get_public_root():
964 """Get the public read-write directory root.
966 This returns a Deferred that fires with an IDirectoryNode instance
967 corresponding to the global shared root directory."""
970 def have_private_root():
971 """Return a Boolean, True if get_public_root() will work."""
972 def get_private_root():
973 """Get the private directory root.
975 This returns a Deferred that fires with an IDirectoryNode instance
976 corresponding to this client's private root directory."""
978 def get_node_at_path(path):
979 """Transform a path into an IDirectoryNode or IFileNode.
981 The path can either be a single string or a list of path-name
982 elements. The former is generated from the latter by using
983 .join('/'). If the first element of this list is '~', the rest will
984 be interpreted relative to the local user's private root directory.
985 Otherwse it will be interpreted relative to the global public root
986 directory. As a result, the following three values of 'path' are
991 ['dirname', 'foo.txt']
993 This method returns a Deferred that fires with the node in question,
994 or errbacks with an IndexError if the target node could not be found.
998 """Transform a URI (or IURI) into an IDirectoryNode or IFileNode.
1000 This returns a Deferred that will fire with an instance that provides
1001 either IDirectoryNode or IFileNode, as appropriate."""
1003 class NotCapableError(Exception):
1004 """You have tried to write to a read-only node."""
1006 class RIControlClient(RemoteInterface):
1008 def wait_for_client_connections(num_clients=int):
1009 """Do not return until we have connections to at least NUM_CLIENTS
1013 def upload_from_file_to_uri(filename=str):
1014 """Upload a file to the grid. This accepts a filename (which must be
1015 absolute) that points to a file on the node's local disk. The node
1016 will read the contents of this file, upload it to the grid, then
1017 return the URI at which it was uploaded.
1021 def download_from_uri_to_file(uri=URI, filename=str):
1022 """Download a file from the grid, placing it on the node's local disk
1023 at the given filename (which must be absolute[?]). Returns the
1024 absolute filename where the file was written."""
1029 def get_memory_usage():
1030 """Return a dict describes the amount of memory currently in use. The
1031 keys are 'VmPeak', 'VmSize', and 'VmData'. The values are integers,
1032 measuring memory consupmtion in bytes."""
1033 return DictOf(str, int)