2 from zope.interface import Interface
3 from foolscap.schema import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
5 from foolscap import RemoteInterface, Referenceable
9 Hash = StringConstraint(maxLength=HASH_SIZE,
10 minLength=HASH_SIZE)# binary format 32-byte SHA256 hash
11 Nodeid = StringConstraint(maxLength=20,
12 minLength=20) # binary format 20-byte SHA1 hash
13 FURL = StringConstraint(1000)
14 StorageIndex = StringConstraint(16)
15 URI = StringConstraint(300) # kind of arbitrary
16 MAX_BUCKETS = 200 # per peer
17 ShareData = StringConstraint(400000) # 1MB segment / k=3 = 334kB
18 URIExtensionData = StringConstraint(1000)
19 LeaseRenewSecret = Hash # used to protect bucket lease renewal requests
20 LeaseCancelSecret = Hash # used to protect bucket lease cancellation requests
23 class RIIntroducerClient(RemoteInterface):
24 def new_peers(furls=SetOf(FURL)):
26 def set_encoding_parameters(parameters=(int, int, int)):
27 """Advise the client of the recommended k-of-n encoding parameters
28 for this grid. 'parameters' is a tuple of (k, desired, n), where 'n'
29 is the total number of shares that will be created for any given
30 file, while 'k' is the number of shares that must be retrieved to
31 recover that file, and 'desired' is the minimum number of shares that
32 must be placed before the uploader will consider its job a success.
33 n/k is the expansion ratio, while k determines the robustness.
35 Introducers should specify 'n' according to the expected size of the
36 grid (there is no point to producing more shares than there are
37 peers), and k according to the desired reliability-vs-overhead goals.
39 Note that setting k=1 is equivalent to simple replication.
43 class RIIntroducer(RemoteInterface):
44 def hello(node=RIIntroducerClient, furl=FURL):
47 class RIClient(RemoteInterface):
49 """Return a tuple of (my_version, oldest_supported) strings.
51 Each string can be parsed by an allmydata.util.version.Version
52 instance, and then compared. The first goal is to make sure that
53 nodes are not confused by speaking to an incompatible peer. The
54 second goal is to enable the development of backwards-compatibility
57 This method is likely to change in incompatible ways until we get the
58 whole compatibility scheme nailed down.
60 return TupleOf(str, str)
61 def get_service(name=str):
66 class RIBucketWriter(RemoteInterface):
67 def write(offset=int, data=ShareData):
72 If the data that has been written is incomplete or inconsistent then
73 the server will throw the data away, else it will store it for future
78 class RIBucketReader(RemoteInterface):
79 def read(offset=int, length=int):
83 class RIStorageServer(RemoteInterface):
84 def allocate_buckets(storage_index=StorageIndex,
85 renew_secret=LeaseRenewSecret,
86 cancel_secret=LeaseCancelSecret,
87 sharenums=SetOf(int, maxLength=MAX_BUCKETS),
88 allocated_size=int, canary=Referenceable):
90 @param storage_index: the index of the bucket to be created or
92 @param sharenums: these are the share numbers (probably between 0 and
93 99) that the sender is proposing to store on this
95 @param renew_secret: This is the secret used to protect bucket refresh
96 This secret is generated by the client and
97 stored for later comparison by the server. Each
98 server is given a different secret.
99 @param cancel_secret: Like renew_secret, but protects bucket decref.
100 @param canary: If the canary is lost before close(), the bucket is
102 @return: tuple of (alreadygot, allocated), where alreadygot is what we
103 already have and is what we hereby agree to accept. New
104 leases are added for shares in both lists.
106 return TupleOf(SetOf(int, maxLength=MAX_BUCKETS),
107 DictOf(int, RIBucketWriter, maxKeys=MAX_BUCKETS))
109 def renew_lease(storage_index=StorageIndex, renew_secret=LeaseRenewSecret):
111 Renew the lease on a given bucket. Some networks will use this, some
115 def cancel_lease(storage_index=StorageIndex,
116 cancel_secret=LeaseCancelSecret):
118 Cancel the lease on a given bucket. If this was the last lease on the
119 bucket, the bucket will be deleted.
122 def get_buckets(storage_index=StorageIndex):
123 return DictOf(int, RIBucketReader, maxKeys=MAX_BUCKETS)
126 class IStorageBucketWriter(Interface):
127 def put_block(segmentnum=int, data=ShareData):
128 """@param data: For most segments, this data will be 'blocksize'
129 bytes in length. The last segment might be shorter.
130 @return: a Deferred that fires (with None) when the operation completes
133 def put_plaintext_hashes(hashes=ListOf(Hash, maxLength=2**20)):
135 @return: a Deferred that fires (with None) when the operation completes
138 def put_crypttext_hashes(hashes=ListOf(Hash, maxLength=2**20)):
140 @return: a Deferred that fires (with None) when the operation completes
143 def put_block_hashes(blockhashes=ListOf(Hash, maxLength=2**20)):
145 @return: a Deferred that fires (with None) when the operation completes
148 def put_share_hashes(sharehashes=ListOf(TupleOf(int, Hash),
151 @return: a Deferred that fires (with None) when the operation completes
154 def put_uri_extension(data=URIExtensionData):
155 """This block of data contains integrity-checking information (hashes
156 of plaintext, crypttext, and shares), as well as encoding parameters
157 that are necessary to recover the data. This is a serialized dict
158 mapping strings to other strings. The hash of this data is kept in
159 the URI and verified before any of the data is used. All buckets for
160 a given file contain identical copies of this data.
162 The serialization format is specified with the following pseudocode:
163 for k in sorted(dict.keys()):
164 assert re.match(r'^[a-zA-Z_\-]+$', k)
165 write(k + ':' + netstring(dict[k]))
167 @return: a Deferred that fires (with None) when the operation completes
171 """Finish writing and close the bucket. The share is not finalized
172 until this method is called: if the uploading client disconnects
173 before calling close(), the partially-written share will be
176 @return: a Deferred that fires (with None) when the operation completes
179 class IStorageBucketReader(Interface):
181 def get_block(blocknum=int):
182 """Most blocks will be the same size. The last block might be shorter
188 def get_plaintext_hashes():
190 @return: ListOf(Hash, maxLength=2**20)
193 def get_crypttext_hashes():
195 @return: ListOf(Hash, maxLength=2**20)
198 def get_block_hashes():
200 @return: ListOf(Hash, maxLength=2**20)
203 def get_share_hashes():
205 @return: ListOf(TupleOf(int, Hash), maxLength=2**20)
208 def get_uri_extension():
210 @return: URIExtensionData
215 # hm, we need a solution for forward references in schemas
216 from foolscap.schema import Any
217 RIMutableDirectoryNode_ = Any() # TODO: how can we avoid this?
219 FileNode_ = Any() # TODO: foolscap needs constraints on copyables
220 DirectoryNode_ = Any() # TODO: same
221 AnyNode_ = ChoiceOf(FileNode_, DirectoryNode_)
224 class RIVirtualDriveServer(RemoteInterface):
225 def get_public_root_uri():
226 """Obtain the URI for this server's global publically-writable root
227 directory. This returns a read-write directory URI.
229 If this vdrive server does not offer a public root, this will
230 raise an exception."""
233 def create_directory(index=Hash, write_enabler=Hash):
234 """Create a new (empty) directory, unattached to anything else.
236 This returns the same index that was passed in.
240 def get(index=Hash, key=Hash):
241 """Retrieve a named child of the given directory. 'index' specifies
242 which directory is being accessed, and is generally the hash of the
243 read key. 'key' is the hash of the read key and the child name.
245 This operation returns a pair of encrypted strings. The first string
246 is meant to be decrypted by the Write Key and provides read-write
247 access to the child. If this directory holds read-only access to the
248 child, this first string will be an empty string. The second string
249 is meant to be decrypted by the Read Key and provides read-only
252 When the child is a read-write directory, the encrypted URI:DIR-RO
253 will be in the read slot, and the encrypted URI:DIR will be in the
254 write slot. When the child is a read-only directory, the encrypted
255 URI:DIR-RO will be in the read slot and the write slot will be empty.
256 When the child is a CHK file, the encrypted URI:CHK will be in the
257 read slot, and the write slot will be empty.
259 This might raise IndexError if there is no child by the desired name.
261 return (EncryptedThing, EncryptedThing)
263 def list(index=Hash):
264 """List the contents of a directory.
266 This returns a list of (NAME, WRITE, READ) tuples. Each value is an
267 encrypted string (although the WRITE value may sometimes be an empty
270 NAME: the child name, encrypted with the Read Key
271 WRITE: the child write URI, encrypted with the Write Key, or an
272 empty string if this child is read-only
273 READ: the child read URI, encrypted with the Read Key
275 return ListOf((EncryptedThing, EncryptedThing, EncryptedThing),
279 def set(index=Hash, write_enabler=Hash, key=Hash,
280 name=EncryptedThing, write=EncryptedThing, read=EncryptedThing):
281 """Set a child object. I will replace any existing child of the same
285 def delete(index=Hash, write_enabler=Hash, key=Hash):
286 """Delete a specific child.
288 This uses the hashed key to locate a specific child, and deletes it.
292 class IURI(Interface):
293 def init_from_string(uri):
294 """Accept a string (as created by my to_string() method) and populate
295 this instance with its data. I am not normally called directly,
296 please use the module-level uri.from_string() function to convert
297 arbitrary URI strings into IURI-providing instances."""
300 """Return False if this URI be used to modify the data. Return True
301 if this URI cannot be used to modify the data."""
304 """Return True if the data can be modified by *somebody* (perhaps
305 someone who has a more powerful URI than this one)."""
308 """Return another IURI instance, which represents a read-only form of
309 this one. If is_readonly() is True, this returns self."""
312 """Return a string of printable ASCII characters, suitable for
313 passing into init_from_string."""
315 class IDirnodeURI(Interface):
316 """I am a URI which represents a dirnode."""
318 class IFileURI(Interface):
319 """I am a URI which represents a filenode."""
321 """Return the length (in bytes) of the file that I represent."""
324 class IFileNode(Interface):
325 def download(target):
326 """Download the file's contents to a given IDownloadTarget"""
327 def download_to_data():
328 """Download the file's contents. Return a Deferred that fires
329 with those contents."""
332 """Return the URI that can be used by others to get access to this
336 """Return the length (in bytes) of the data this node represents."""
338 def get_refresh_capability():
339 """Return a string that represents the 'refresh capability' for this
340 node. The holder of this capability will be able to renew the lease
341 for this node, protecting it from garbage-collection.
344 class IDirectoryNode(Interface):
346 """Return True if this directory is mutable, False if it is read-only.
350 """Return the directory URI that can be used by others to get access
351 to this directory node. If this node is read-only, the URI will only
352 offer read-only access. If this node is read-write, the URI will
353 offer read-write acess.
355 If you have read-write access to a directory and wish to share merely
356 read-only access with others, use get_immutable_uri().
358 The dirnode ('1') URI returned by this method can be used in
359 set_uri() on a different directory ('2') to 'mount' a reference to
360 this directory ('1') under the other ('2'). This URI is just a
361 string, so it can be passed around through email or other out-of-band
365 def get_immutable_uri():
366 """Return the directory URI that can be used by others to get
367 read-only access to this directory node. The result is a read-only
368 URI, regardless of whether this dirnode is read-only or read-write.
370 If you have merely read-only access to this dirnode,
371 get_immutable_uri() will return the same thing as get_uri().
374 def get_refresh_capability():
375 """Return a string that represents the 'refresh capability' for this
376 node. The holder of this capability will be able to renew the lease
377 for this node, protecting it from garbage-collection.
381 """I return a Deferred that fires with a dictionary mapping child
382 name to an IFileNode or IDirectoryNode."""
385 """I return a Deferred that fires with a boolean, True if there
386 exists a child of the given name, False if not."""
389 """I return a Deferred that fires with a specific named child node,
390 either an IFileNode or an IDirectoryNode."""
392 def get_child_at_path(path):
393 """Transform a child path into an IDirectoryNode or IFileNode.
395 I perform a recursive series of 'get' operations to find the named
396 descendant node. I return a Deferred that fires with the node, or
397 errbacks with IndexError if the node could not be found.
399 The path can be either a single string (slash-separated) or a list of
403 def set_uri(name, child_uri):
404 """I add a child (by URI) at the specific name. I return a Deferred
405 that fires when the operation finishes. I will replace any existing
406 child of the same name.
408 The child_uri could be for a file, or for a directory (either
409 read-write or read-only, using a URI that came from get_uri() ).
411 If this directory node is read-only, the Deferred will errback with a
414 def set_node(name, child):
415 """I add a child at the specific name. I return a Deferred that fires
416 when the operation finishes. This Deferred will fire with the child
417 node that was just added. I will replace any existing child of the
420 If this directory node is read-only, the Deferred will errback with a
423 def add_file(name, uploadable):
424 """I upload a file (using the given IUploadable), then attach the
425 resulting FileNode to the directory at the given name. I return a
426 Deferred that fires (with the IFileNode of the uploaded file) when
427 the operation completes."""
430 """I remove the child at the specific name. I return a Deferred that
431 fires when the operation finishes."""
433 def create_empty_directory(name):
434 """I create and attach an empty directory at the given name. I return
435 a Deferred that fires when the operation finishes."""
437 def move_child_to(current_child_name, new_parent, new_child_name=None):
438 """I take one of my children and move them to a new parent. The child
439 is referenced by name. On the new parent, the child will live under
440 'new_child_name', which defaults to 'current_child_name'. I return a
441 Deferred that fires when the operation finishes."""
443 def build_manifest():
444 """Return a set of refresh-capabilities for all nodes (directories
445 and files) reachable from this one."""
447 class ICodecEncoder(Interface):
448 def set_params(data_size, required_shares, max_shares):
449 """Set up the parameters of this encoder.
451 This prepares the encoder to perform an operation that converts a
452 single block of data into a number of shares, such that a future
453 ICodecDecoder can use a subset of these shares to recover the
454 original data. This operation is invoked by calling encode(). Once
455 the encoding parameters are set up, the encode operation can be
456 invoked multiple times.
458 set_params() prepares the encoder to accept blocks of input data that
459 are exactly 'data_size' bytes in length. The encoder will be prepared
460 to produce 'max_shares' shares for each encode() operation (although
461 see the 'desired_share_ids' to use less CPU). The encoding math will
462 be chosen such that the decoder can get by with as few as
463 'required_shares' of these shares and still reproduce the original
464 data. For example, set_params(1000, 5, 5) offers no redundancy at
465 all, whereas set_params(1000, 1, 10) provides 10x redundancy.
467 Numerical Restrictions: 'data_size' is required to be an integral
468 multiple of 'required_shares'. In general, the caller should choose
469 required_shares and max_shares based upon their reliability
470 requirements and the number of peers available (the total storage
471 space used is roughly equal to max_shares*data_size/required_shares),
472 then choose data_size to achieve the memory footprint desired (larger
473 data_size means more efficient operation, smaller data_size means
474 smaller memory footprint).
476 In addition, 'max_shares' must be equal to or greater than
477 'required_shares'. Of course, setting them to be equal causes
478 encode() to degenerate into a particularly slow form of the 'split'
481 See encode() for more details about how these parameters are used.
483 set_params() must be called before any other ICodecEncoder methods
487 def get_encoder_type():
488 """Return a short string that describes the type of this encoder.
490 There is required to be a global table of encoder classes. This method
491 returns an index into this table; the value at this index is an
492 encoder class, and this encoder is an instance of that class.
495 def get_serialized_params(): # TODO: maybe, maybe not
496 """Return a string that describes the parameters of this encoder.
498 This string can be passed to the decoder to prepare it for handling
499 the encoded shares we create. It might contain more information than
500 was presented to set_params(), if there is some flexibility of
503 This string is intended to be embedded in the URI, so there are
504 several restrictions on its contents. At the moment I'm thinking that
505 this means it may contain hex digits and hyphens, and nothing else.
506 The idea is that the URI contains something like '%s:%s:%s' %
507 (encoder.get_encoder_name(), encoder.get_serialized_params(),
508 b2a(crypttext_hash)), and this is enough information to construct a
512 def get_block_size():
513 """Return the length of the shares that encode() will produce.
516 def encode_proposal(data, desired_share_ids=None):
519 'data' must be a string (or other buffer object), and len(data) must
520 be equal to the 'data_size' value passed earlier to set_params().
522 This will return a Deferred that will fire with two lists. The first
523 is a list of shares, each of which is a string (or other buffer
524 object) such that len(share) is the same as what get_share_size()
525 returned earlier. The second is a list of shareids, in which each is
526 an integer. The lengths of the two lists will always be equal to each
527 other. The user should take care to keep each share closely
528 associated with its shareid, as one is useless without the other.
530 The length of this output list will normally be the same as the value
531 provided to the 'max_shares' parameter of set_params(). This may be
532 different if 'desired_share_ids' is provided.
534 'desired_share_ids', if provided, is required to be a sequence of
535 ints, each of which is required to be >= 0 and < max_shares. If not
536 provided, encode() will produce 'max_shares' shares, as if
537 'desired_share_ids' were set to range(max_shares). You might use this
538 if you initially thought you were going to use 10 peers, started
539 encoding, and then two of the peers dropped out: you could use
540 desired_share_ids= to skip the work (both memory and CPU) of
541 producing shares for the peers which are no longer available.
545 def encode(inshares, desired_share_ids=None):
546 """Encode some data. This may be called multiple times. Each call is
549 inshares is a sequence of length required_shares, containing buffers
550 (i.e. strings), where each buffer contains the next contiguous
551 non-overlapping segment of the input data. Each buffer is required to
552 be the same length, and the sum of the lengths of the buffers is
553 required to be exactly the data_size promised by set_params(). (This
554 implies that the data has to be padded before being passed to
555 encode(), unless of course it already happens to be an even multiple
556 of required_shares in length.)
558 ALSO: the requirement to break up your data into 'required_shares'
559 chunks before calling encode() feels a bit surprising, at least from
560 the point of view of a user who doesn't know how FEC works. It feels
561 like an implementation detail that has leaked outside the
562 abstraction barrier. Can you imagine a use case in which the data to
563 be encoded might already be available in pre-segmented chunks, such
564 that it is faster or less work to make encode() take a list rather
565 than splitting a single string?
567 ALSO ALSO: I think 'inshares' is a misleading term, since encode()
568 is supposed to *produce* shares, so what it *accepts* should be
569 something other than shares. Other places in this interface use the
570 word 'data' for that-which-is-not-shares.. maybe we should use that
573 'desired_share_ids', if provided, is required to be a sequence of
574 ints, each of which is required to be >= 0 and < max_shares. If not
575 provided, encode() will produce 'max_shares' shares, as if
576 'desired_share_ids' were set to range(max_shares). You might use this
577 if you initially thought you were going to use 10 peers, started
578 encoding, and then two of the peers dropped out: you could use
579 desired_share_ids= to skip the work (both memory and CPU) of
580 producing shares for the peers which are no longer available.
582 For each call, encode() will return a Deferred that fires with two
583 lists, one containing shares and the other containing the shareids.
584 The get_share_size() method can be used to determine the length of
585 the share strings returned by encode(). Each shareid is a small
586 integer, exactly as passed into 'desired_share_ids' (or
587 range(max_shares), if desired_share_ids was not provided).
589 The shares and their corresponding shareids are required to be kept
590 together during storage and retrieval. Specifically, the share data is
591 useless by itself: the decoder needs to be told which share is which
592 by providing it with both the shareid and the actual share data.
594 This function will allocate an amount of memory roughly equal to::
596 (max_shares - required_shares) * get_share_size()
598 When combined with the memory that the caller must allocate to
599 provide the input data, this leads to a memory footprint roughly
600 equal to the size of the resulting encoded shares (i.e. the expansion
601 factor times the size of the input segment).
606 # returning a list of (shareidN,shareN) tuples instead of a pair of
607 # lists (shareids..,shares..). Brian thought the tuples would
608 # encourage users to keep the share and shareid together throughout
609 # later processing, Zooko pointed out that the code to iterate
610 # through two lists is not really more complicated than using a list
611 # of tuples and there's also a performance improvement
613 # having 'data_size' not required to be an integral multiple of
614 # 'required_shares'. Doing this would require encode() to perform
615 # padding internally, and we'd prefer to have any padding be done
616 # explicitly by the caller. Yes, it is an abstraction leak, but
617 # hopefully not an onerous one.
620 class ICodecDecoder(Interface):
621 def set_serialized_params(params):
622 """Set up the parameters of this encoder, from a string returned by
623 encoder.get_serialized_params()."""
625 def get_needed_shares():
626 """Return the number of shares needed to reconstruct the data.
627 set_serialized_params() is required to be called before this."""
629 def decode(some_shares, their_shareids):
630 """Decode a partial list of shares into data.
632 'some_shares' is required to be a sequence of buffers of sharedata, a
633 subset of the shares returned by ICodecEncode.encode(). Each share is
634 required to be of the same length. The i'th element of their_shareids
635 is required to be the shareid of the i'th buffer in some_shares.
637 This returns a Deferred which fires with a sequence of buffers. This
638 sequence will contain all of the segments of the original data, in
639 order. The sum of the lengths of all of the buffers will be the
640 'data_size' value passed into the original ICodecEncode.set_params()
641 call. To get back the single original input block of data, use
642 ''.join(output_buffers), or you may wish to simply write them in
643 order to an output file.
645 Note that some of the elements in the result sequence may be
646 references to the elements of the some_shares input sequence. In
647 particular, this means that if those share objects are mutable (e.g.
648 arrays) and if they are changed, then both the input (the
649 'some_shares' parameter) and the output (the value given when the
650 deferred is triggered) will change.
652 The length of 'some_shares' is required to be exactly the value of
653 'required_shares' passed into the original ICodecEncode.set_params()
657 class IEncoder(Interface):
658 """I take an object that provides IEncryptedUploadable, which provides
659 encrypted data, and a list of shareholders. I then encode, hash, and
660 deliver shares to those shareholders. I will compute all the necessary
661 Merkle hash trees that are necessary to validate the crypttext that
662 eventually comes back from the shareholders. I provide the URI Extension
663 Block Hash, and the encoding parameters, both of which must be included
666 I do not choose shareholders, that is left to the IUploader. I must be
667 given a dict of RemoteReferences to storage buckets that are ready and
668 willing to receive data.
672 """Specify the number of bytes that will be encoded. This must be
673 peformed before get_serialized_params() can be called.
675 def set_params(params):
676 """Override the default encoding parameters. 'params' is a tuple of
677 (k,d,n), where 'k' is the number of required shares, 'd' is the
678 shares_of_happiness, and 'n' is the total number of shares that will
681 Encoding parameters can be set in three ways. 1: The Encoder class
682 provides defaults (25/75/100). 2: the Encoder can be constructed with
683 an 'options' dictionary, in which the
684 needed_and_happy_and_total_shares' key can be a (k,d,n) tuple. 3:
685 set_params((k,d,n)) can be called.
687 If you intend to use set_params(), you must call it before
688 get_share_size or get_param are called.
691 def set_encrypted_uploadable(u):
692 """Provide a source of encrypted upload data. 'u' must implement
693 IEncryptedUploadable.
695 When this is called, the IEncryptedUploadable will be queried for its
696 length and the storage_index that should be used.
698 This returns a Deferred that fires with this Encoder instance.
700 This must be performed before start() can be called.
704 """Return an encoding parameter, by name.
706 'storage_index': return a string with the (16-byte truncated SHA-256
707 hash) storage index to which these shares should be
710 'share_counts': return a tuple describing how many shares are used:
711 (needed_shares, shares_of_happiness, total_shares)
713 'num_segments': return an int with the number of segments that
716 'segment_size': return an int with the size of each segment.
718 'block_size': return the size of the individual blocks that will
719 be delivered to a shareholder's put_block() method. By
720 knowing this, the shareholder will be able to keep all
721 blocks in a single file and still provide random access
722 when reading them. # TODO: can we avoid exposing this?
724 'share_size': an int with the size of the data that will be stored
725 on each shareholder. This is aggregate amount of data
726 that will be sent to the shareholder, summed over all
727 the put_block() calls I will ever make. It is useful to
728 determine this size before asking potential
729 shareholders whether they will grant a lease or not,
730 since their answers will depend upon how much space we
731 need. TODO: this might also include some amount of
732 overhead, like the size of all the hashes. We need to
733 decide whether this is useful or not.
735 'serialized_params': a string with a concise description of the
736 codec name and its parameters. This may be passed
737 into the IUploadable to let it make sure that
738 the same file encoded with different parameters
739 will result in different storage indexes.
741 Once this is called, set_size() and set_params() may not be called.
744 def set_shareholders(shareholders):
745 """Tell the encoder where to put the encoded shares. 'shareholders'
746 must be a dictionary that maps share number (an integer ranging from
747 0 to n-1) to an instance that provides IStorageBucketWriter. This
748 must be performed before start() can be called."""
751 """Begin the encode/upload process. This involves reading encrypted
752 data from the IEncryptedUploadable, encoding it, uploading the shares
753 to the shareholders, then sending the hash trees.
755 set_encrypted_uploadable() and set_shareholders() must be called
756 before this can be invoked.
758 This returns a Deferred that fires with a tuple of
759 (uri_extension_hash, needed_shares, total_shares, size) when the
760 upload process is complete. This information, plus the encryption
761 key, is sufficient to construct the URI.
764 class IDecoder(Interface):
765 """I take a list of shareholders and some setup information, then
766 download, validate, decode, and decrypt data from them, writing the
767 results to an output file.
769 I do not locate the shareholders, that is left to the IDownloader. I must
770 be given a dict of RemoteReferences to storage buckets that are ready to
775 """I take a file-like object (providing write and close) to which all
776 the plaintext data will be written.
778 TODO: producer/consumer . Maybe write() should return a Deferred that
779 indicates when it will accept more data? But probably having the
780 IDecoder be a producer is easier to glue to IConsumer pieces.
783 def set_shareholders(shareholders):
784 """I take a dictionary that maps share identifiers (small integers)
785 to RemoteReferences that provide RIBucketReader. This must be called
789 """I start the download. This process involves retrieving data and
790 hash chains from the shareholders, using the hashes to validate the
791 data, decoding the shares into segments, decrypting the segments,
792 then writing the resulting plaintext to the output file.
794 I return a Deferred that will fire (with self) when the download is
798 class IDownloadTarget(Interface):
800 """Called before any calls to write() or close(). If an error
801 occurs before any data is available, fail() may be called without
802 a previous call to open().
804 'size' is the length of the file being downloaded, in bytes."""
807 """Output some data to the target."""
809 """Inform the target that there is no more data to be written."""
811 """fail() is called to indicate that the download has failed. 'why'
812 is a Failure object indicating what went wrong. No further methods
813 will be invoked on the IDownloadTarget after fail()."""
814 def register_canceller(cb):
815 """The FileDownloader uses this to register a no-argument function
816 that the target can call to cancel the download. Once this canceller
817 is invoked, no further calls to write() or close() will be made."""
819 """When the FileDownloader is done, this finish() function will be
820 called. Whatever it returns will be returned to the invoker of
824 class IDownloader(Interface):
825 def download(uri, target):
826 """Perform a CHK download, sending the data to the given target.
827 'target' must provide IDownloadTarget.
829 Returns a Deferred that fires (with the results of target.finish)
830 when the download is finished, or errbacks if something went wrong."""
832 class IEncryptedUploadable(Interface):
834 """This behaves just like IUploadable.get_size()."""
836 def set_serialized_encoding_parameters(serialized_encoding_parameters):
837 """Tell me what encoding parameters will be used for my data.
839 'serialized_encoding_parameters' is a string which indicates how the
840 data will be encoded (codec name, blocksize, number of shares).
842 I may use this when get_storage_index() is called, to influence the
843 index that I return. Or, I may just ignore it.
845 set_serialized_encoding_parameters() may be called 0 or 1 times. If
846 called, it must be called before get_storage_index().
849 def get_storage_index():
850 """Return a Deferred that fires with a 16-byte storage index. This
851 value may be influenced by the parameters earlier set by
852 set_serialized_encoding_parameters().
855 def set_segment_size(segment_size):
856 """Set the segment size, to allow the IEncryptedUploadable to
857 accurately create the plaintext segment hash tree. This must be
858 called before any calls to read_encrypted."""
860 def read_encrypted(length):
861 """This behaves just like IUploadable.read(), but returns crypttext
862 instead of plaintext. set_segment_size() must be called before the
863 first call to read_encrypted()."""
865 def get_plaintext_segment_hashtree_nodes(num_segments):
866 """Get the nodes of a merkle hash tree over the plaintext segments.
868 This returns a Deferred which fires with a sequence of hashes. Each
869 hash is a node of a merkle hash tree, generally obtained from::
871 tuple(HashTree(segment_hashes))
873 'num_segments' is used to assert that the number of segments that the
874 IEncryptedUploadable handled matches the number of segments that the
875 encoder was expecting.
878 def get_plaintext_hash():
879 """Get the hash of the whole plaintext.
881 This returns a Deferred which fires with a tagged SHA-256 hash of the
882 whole plaintext, obtained from hashutil.plaintext_hash(data).
886 """Just like IUploadable.close()."""
888 class IUploadable(Interface):
890 """Return a Deferred that will fire with the length of the data to be
891 uploaded, in bytes. This will be called before the data is actually
892 used, to compute encoding parameters.
895 def set_serialized_encoding_parameters(serialized_encoding_parameters):
896 """Tell me what encoding parameters will be used for my data.
898 'serialized_encoding_parameters' is a string which indicates how the
899 data will be encoded (codec name, blocksize, number of shares).
901 I may use this when get_encryption_key() is called, to influence the
902 key that I return. Or, I may just ignore it.
904 set_serialized_encoding_parameters() may be called 0 or 1 times. If
905 called, it must be called before get_encryption_key().
908 def get_encryption_key():
909 """Return a Deferred that fires with a 16-byte AES key. This key will
910 be used to encrypt the data. The key will also be hashed to derive
913 Uploadables which want to achieve convergence should hash their file
914 contents and the serialized_encoding_parameters to form the key
915 (which of course requires a full pass over the data). Uploadables can
916 use the upload.ConvergentUploadMixin class to achieve this
919 Uploadables which do not care about convergence (or do not wish to
920 make multiple passes over the data) can simply return a
921 strongly-random 16 byte string.
923 get_encryption_key() may be called multiple times: the IUploadable is
924 required to return the same value each time.
928 """Return a Deferred that fires with a list of strings (perhaps with
929 only a single element) which, when concatenated together, contain the
930 next 'length' bytes of data. If EOF is near, this may provide fewer
931 than 'length' bytes. The total number of bytes provided by read()
932 before it signals EOF must equal the size provided by get_size().
934 If the data must be acquired through multiple internal read
935 operations, returning a list instead of a single string may help to
936 reduce string copies.
938 'length' will typically be equal to (min(get_size(),1MB)/req_shares),
939 so a 10kB file means length=3kB, 100kB file means length=30kB,
940 and >=1MB file means length=300kB.
942 This method provides for a single full pass through the data. Later
943 use cases may desire multiple passes or access to only parts of the
944 data (such as a mutable file making small edits-in-place). This API
945 will be expanded once those use cases are better understood.
949 """The upload is finished, and whatever filehandle was in use may be
952 class IUploader(Interface):
953 def upload(uploadable):
954 """Upload the file. 'uploadable' must impement IUploadable. This
955 returns a Deferred which fires with the URI of the file."""
957 def upload_ssk(write_capability, new_version, uploadable):
958 """TODO: how should this work?"""
959 def upload_data(data):
960 """Like upload(), but accepts a string."""
962 def upload_filename(filename):
963 """Like upload(), but accepts an absolute pathname."""
965 def upload_filehandle(filehane):
966 """Like upload(), but accepts an open filehandle."""
968 class IVirtualDrive(Interface):
969 """I am a service that may be available to a client.
971 Within any client program, this service can be retrieved by using
972 client.getService('vdrive').
975 def have_public_root():
976 """Return a Boolean, True if get_public_root() will work."""
977 def get_public_root():
978 """Get the public read-write directory root.
980 This returns a Deferred that fires with an IDirectoryNode instance
981 corresponding to the global shared root directory."""
984 def have_private_root():
985 """Return a Boolean, True if get_public_root() will work."""
986 def get_private_root():
987 """Get the private directory root.
989 This returns a Deferred that fires with an IDirectoryNode instance
990 corresponding to this client's private root directory."""
992 def get_node_at_path(path):
993 """Transform a path into an IDirectoryNode or IFileNode.
995 The path can either be a single string or a list of path-name
996 elements. The former is generated from the latter by using
997 .join('/'). If the first element of this list is '~', the rest will
998 be interpreted relative to the local user's private root directory.
999 Otherwse it will be interpreted relative to the global public root
1000 directory. As a result, the following three values of 'path' are
1005 ['dirname', 'foo.txt']
1007 This method returns a Deferred that fires with the node in question,
1008 or errbacks with an IndexError if the target node could not be found.
1012 """Transform a URI (or IURI) into an IDirectoryNode or IFileNode.
1014 This returns a Deferred that will fire with an instance that provides
1015 either IDirectoryNode or IFileNode, as appropriate."""
1017 class NotCapableError(Exception):
1018 """You have tried to write to a read-only node."""
1020 class RIControlClient(RemoteInterface):
1022 def wait_for_client_connections(num_clients=int):
1023 """Do not return until we have connections to at least NUM_CLIENTS
1027 def upload_from_file_to_uri(filename=str):
1028 """Upload a file to the grid. This accepts a filename (which must be
1029 absolute) that points to a file on the node's local disk. The node
1030 will read the contents of this file, upload it to the grid, then
1031 return the URI at which it was uploaded.
1035 def download_from_uri_to_file(uri=URI, filename=str):
1036 """Download a file from the grid, placing it on the node's local disk
1037 at the given filename (which must be absolute[?]). Returns the
1038 absolute filename where the file was written."""
1043 def get_memory_usage():
1044 """Return a dict describes the amount of memory currently in use. The
1045 keys are 'VmPeak', 'VmSize', and 'VmData'. The values are integers,
1046 measuring memory consupmtion in bytes."""
1047 return DictOf(str, int)