Number = IntegerConstraint(8) # 2**(8*8) == 16EiB ~= 18e18 ~= 18 exabytes
Offset = Number
ReadSize = int # the 'int' constraint is 2**31 == 2Gib -- large files are processed in not-so-large increments
-WriteEnablerSecret = Hash # used to protect mutable bucket modifications
-LeaseRenewSecret = Hash # used to protect bucket lease renewal requests
-LeaseCancelSecret = Hash # used to protect bucket lease cancellation requests
+WriteEnablerSecret = Hash # used to protect mutable share modifications
+LeaseRenewSecret = Hash # used to protect lease renewal requests
+LeaseCancelSecret = Hash # was used to protect lease cancellation requests
+
class RIBucketWriter(RemoteInterface):
""" Objects of this kind live on the server side. """
failures. I will record their concern so that my operator can
manually inspect the shares in question. I return None.
- This is a wrapper around RIStorageServer.advise_corrupt_share(),
- which is tied to a specific share, and therefore does not need the
+ This is a wrapper around RIStorageServer.advise_corrupt_share()
+ that is tied to a specific share, and therefore does not need the
extra share-identifying arguments. Please see that method for full
documentation.
"""
This secret is generated by the client and
stored for later comparison by the server. Each
server is given a different secret.
- @param cancel_secret: Like renew_secret, but protects bucket decref.
+ @param cancel_secret: This no longer allows lease cancellation, but
+ must still be a unique value identifying the
+ lease. XXX stop relying on it to be unique.
@param canary: If the canary is lost before close(), the bucket is
deleted.
@return: tuple of (alreadygot, allocated), where alreadygot is what we
For mutable shares, if the given renew_secret does not match an
existing lease, IndexError will be raised with a note listing the
server-nodeids on the existing leases, so leases on migrated shares
- can be renewed or cancelled. For immutable shares, IndexError
- (without the note) will be raised.
+ can be renewed. For immutable shares, IndexError (without the note)
+ will be raised.
"""
return Any()
This secret is generated by the client and
stored for later comparison by the server. Each
server is given a different secret.
- @param cancel_secret: Like renew_secret, but protects bucket decref.
+ @param cancel_secret: This no longer allows lease cancellation, but
+ must still be a unique value identifying the
+ lease. XXX stop relying on it to be unique.
The 'secrets' argument is a tuple of (write_enabler, renew_secret,
cancel_secret). The first is required to perform any write. The
(binary) storage index string, and 'shnum' is the integer share
number. 'reason' is a human-readable explanation of the problem,
probably including some expected hash values and the computed ones
- which did not match. Corruption advisories for mutable shares should
+ that did not match. Corruption advisories for mutable shares should
include a hash of the public key (the same value that appears in the
mutable-file verify-cap), since the current share format does not
store that on disk.
remote_host: the IAddress, if connected, otherwise None
This method is intended for monitoring interfaces, such as a web page
- which describes connecting and connected peers.
+ that describes connecting and connected peers.
"""
def get_all_peerids():
"""
The interface for a writer around a mutable slot on a remote server.
"""
- def set_checkstring(checkstring, *args):
+ def set_checkstring(seqnum_or_checkstring, root_hash=None, salt=None):
"""
Set the checkstring that I will pass to the remote server when
writing.
# TODO: rename to get_read_cap()
def get_readonly():
- """Return another IURI instance, which represents a read-only form of
+ """Return another IURI instance that represents a read-only form of
this one. If is_readonly() is True, this returns self."""
def get_verify_cap():
class IDirnodeURI(Interface):
- """I am a URI which represents a dirnode."""
+ """I am a URI that represents a dirnode."""
+
class IFileURI(Interface):
- """I am a URI which represents a filenode."""
+ """I am a URI that represents a filenode."""
def get_size():
"""Return the length (in bytes) of the file that I represent."""
writer-visible data using this writekey.
"""
- # TODO: Can this be overwrite instead of replace?
- def replace(new_contents):
+ def overwrite(new_contents):
"""Replace the contents of the mutable file, provided that no other
node has published (or is attempting to publish, concurrently) a
newer version of the file than this one.
class IFileNode(IFilesystemNode):
- """I am a node which represents a file: a sequence of bytes. I am not a
+ """I am a node that represents a file: a sequence of bytes. I am not a
container, like IDirectoryNode."""
def get_best_readable_version():
"""Return a Deferred that fires with an IReadable for the 'best'
multiple versions of a file present in the grid, some of which might be
unrecoverable (i.e. have fewer than 'k' shares). These versions are
loosely ordered: each has a sequence number and a hash, and any version
- with seqnum=N was uploaded by a node which has seen at least one version
+ with seqnum=N was uploaded by a node that has seen at least one version
with seqnum=N-1.
The 'servermap' (an instance of IMutableFileServerMap) is used to
as a guide to where the shares are located.
I return a Deferred that fires with the requested contents, or
- errbacks with UnrecoverableFileError. Note that a servermap which was
+ errbacks with UnrecoverableFileError. Note that a servermap that was
updated with MODE_ANYTHING or MODE_READ may not know about shares for
all versions (those modes stop querying servers as soon as they can
fulfil their goals), so you may want to use MODE_CHECK (which checks
"""Upload was unable to satisfy 'servers_of_happiness'"""
class UnableToFetchCriticalDownloadDataError(Exception):
- """I was unable to fetch some piece of critical data which is supposed to
+ """I was unable to fetch some piece of critical data that is supposed to
be identically present in all shares."""
class NoServersError(Exception):
exists, and overwrite= was set to False."""
class NoSuchChildError(Exception):
- """A directory node was asked to fetch a child which does not exist."""
+ """A directory node was asked to fetch a child that does not exist."""
def __str__(self):
# avoid UnicodeEncodeErrors when converting to str
return self.__repr__()
is a file, or if must_be_file is True and the child is a directory,
I raise ChildOfWrongTypeError."""
- def create_subdirectory(name, initial_children={}, overwrite=True, metadata=None):
+ def create_subdirectory(name, initial_children={}, overwrite=True,
+ mutable=True, mutable_version=None, metadata=None):
"""I create and attach a directory at the given name. The new
directory can be empty, or it can be populated with children
according to 'initial_children', which takes a dictionary in the same
if you initially thought you were going to use 10 peers, started
encoding, and then two of the peers dropped out: you could use
desired_share_ids= to skip the work (both memory and CPU) of
- producing shares for the peers which are no longer available.
+ producing shares for the peers that are no longer available.
"""
if you initially thought you were going to use 10 peers, started
encoding, and then two of the peers dropped out: you could use
desired_share_ids= to skip the work (both memory and CPU) of
- producing shares for the peers which are no longer available.
+ producing shares for the peers that are no longer available.
For each call, encode() will return a Deferred that fires with two
lists, one containing shares and the other containing the shareids.
required to be of the same length. The i'th element of their_shareids
is required to be the shareid of the i'th buffer in some_shares.
- This returns a Deferred which fires with a sequence of buffers. This
+ This returns a Deferred that fires with a sequence of buffers. This
sequence will contain all of the segments of the original data, in
order. The sum of the lengths of all of the buffers will be the
'data_size' value passed into the original ICodecEncode.set_params()
"""Specify the number of bytes that will be encoded. This must be
peformed before get_serialized_params() can be called.
"""
- def set_params(params):
- """Override the default encoding parameters. 'params' is a tuple of
- (k,d,n), where 'k' is the number of required shares, 'd' is the
- servers_of_happiness, and 'n' is the total number of shares that will
- be created.
-
- Encoding parameters can be set in three ways. 1: The Encoder class
- provides defaults (3/7/10). 2: the Encoder can be constructed with
- an 'options' dictionary, in which the
- needed_and_happy_and_total_shares' key can be a (k,d,n) tuple. 3:
- set_params((k,d,n)) can be called.
-
- If you intend to use set_params(), you must call it before
- get_share_size or get_param are called.
- """
def set_encrypted_uploadable(u):
"""Provide a source of encrypted upload data. 'u' must implement
resuming an interrupted upload (where we need to compute the
plaintext hashes, but don't need the redundant encrypted data)."""
- def get_plaintext_hashtree_leaves(first, last, num_segments):
- """OBSOLETE; Get the leaf nodes of a merkle hash tree over the
- plaintext segments, i.e. get the tagged hashes of the given segments.
- The segment size is expected to be generated by the
- IEncryptedUploadable before any plaintext is read or ciphertext
- produced, so that the segment hashes can be generated with only a
- single pass.
-
- This returns a Deferred which fires with a sequence of hashes, using:
-
- tuple(segment_hashes[first:last])
-
- 'num_segments' is used to assert that the number of segments that the
- IEncryptedUploadable handled matches the number of segments that the
- encoder was expecting.
-
- This method must not be called until the final byte has been read
- from read_encrypted(). Once this method is called, read_encrypted()
- can never be called again.
- """
-
- def get_plaintext_hash():
- """OBSOLETE; Get the hash of the whole plaintext.
-
- This returns a Deferred which fires with a tagged SHA-256 hash of the
- whole plaintext, obtained from hashutil.plaintext_hash(data).
- """
-
def close():
"""Just like IUploadable.close()."""
be used to encrypt the data. The key will also be hashed to derive
the StorageIndex.
- Uploadables which want to achieve convergence should hash their file
+ Uploadables that want to achieve convergence should hash their file
contents and the serialized_encoding_parameters to form the key
(which of course requires a full pass over the data). Uploadables can
use the upload.ConvergentUploadMixin class to achieve this
automatically.
- Uploadables which do not care about convergence (or do not wish to
+ Uploadables that do not care about convergence (or do not wish to
make multiple passes over the data) can simply return a
strongly-random 16 byte string.
def read(length):
"""Return a Deferred that fires with a list of strings (perhaps with
- only a single element) which, when concatenated together, contain the
+ only a single element) that, when concatenated together, contain the
next 'length' bytes of data. If EOF is near, this may provide fewer
than 'length' bytes. The total number of bytes provided by read()
before it signals EOF must equal the size provided by get_size().
def read(length):
"""
- Returns a list of strings which, when concatenated, are the next
+ Returns a list of strings that, when concatenated, are the next
length bytes of the file, or fewer if there are fewer bytes
between the current location and the end of the file.
"""
class IDownloadResults(Interface):
"""I am created internally by download() methods. I contain a number of
- public attributes which contain details about the download process.::
+ public attributes that contain details about the download process.::
.file_size : the size of the file, in bytes
.servers_used : set of server peerids that were used during download
class IUploader(Interface):
def upload(uploadable):
"""Upload the file. 'uploadable' must impement IUploadable. This
- returns a Deferred which fires with an IUploadResults instance, from
+ returns a Deferred that fires with an IUploadResults instance, from
which the URI of the file can be obtained as results.uri ."""
- def upload_ssk(write_capability, new_version, uploadable):
- """TODO: how should this work?"""
class ICheckable(Interface):
def check(monitor, verify=False, add_lease=False):
kind of lease that is obtained (which account number to claim, etc).
TODO: any problems seen during checking will be reported to the
- health-manager.furl, a centralized object which is responsible for
+ health-manager.furl, a centralized object that is responsible for
figuring out why files are unhealthy so corrective action can be
taken.
"""
will be put in the check-and-repair results. The Deferred will not
fire until the repair is complete.
- This returns a Deferred which fires with an instance of
+ This returns a Deferred that fires with an instance of
ICheckAndRepairResults."""
not. Unrecoverable files are obviously unhealthy. Non-distributed LIT
files always return True."""
- def needs_rebalancing():
- """Return a boolean, True if the file/dirs reliability could be
- improved by moving shares to new servers. Non-distributed LIT files
- always return False."""
-
# the following methods all return None for non-distributed LIT files
+ def get_happiness():
+ """Return the happiness count of the file."""
+
def get_encoding_needed():
- """Return 'k', the number of shares required for recovery"""
+ """Return 'k', the number of shares required for recovery."""
def get_encoding_expected():
- """Return 'N', the number of total shares generated"""
+ """Return 'N', the number of total shares generated."""
def get_share_counter_good():
"""Return the number of distinct good shares that were found. For
(IServer, storage_index, sharenum)."""
def get_servers_responding():
- """Return a list of IServer objects, one for each server which
+ """Return a list of IServer objects, one for each server that
responded to the share query (even if they said they didn't have
shares, and even if they said they did have shares but then didn't
send them when asked, or dropped the connection, or returned a
Tahoe process will typically have a single NodeMaker, but unit tests may
create simplified/mocked forms for testing purposes.
"""
- def create_from_cap(writecap, readcap=None, **kwargs):
+
+ def create_from_cap(writecap, readcap=None, deep_immutable=False, name=u"<unknown name>"):
"""I create an IFilesystemNode from the given writecap/readcap. I can
only provide nodes for existing file/directory objects: use my other
methods to create new objects. I return synchronously."""
def create_mutable_file(contents=None, keysize=None):
- """I create a new mutable file, and return a Deferred which will fire
+ """I create a new mutable file, and return a Deferred that will fire
with the IMutableFileNode instance when it is ready. If contents= is
provided (a bytestring), it will be used as the initial contents of
the new file, otherwise the file will contain zero bytes. keysize= is
usual."""
def create_new_mutable_directory(initial_children={}):
- """I create a new mutable directory, and return a Deferred which will
+ """I create a new mutable directory, and return a Deferred that will
fire with the IDirectoryNode instance when it is ready. If
initial_children= is provided (a dict mapping unicode child name to
(childnode, metadata_dict) tuples), the directory will be populated
class IClientStatus(Interface):
def list_all_uploads():
- """Return a list of uploader objects, one for each upload which
+ """Return a list of uploader objects, one for each upload that
currently has an object available (tracked with weakrefs). This is
intended for debugging purposes."""
started uploads."""
def list_all_downloads():
- """Return a list of downloader objects, one for each download which
+ """Return a list of downloader objects, one for each download that
currently has an object available (tracked with weakrefs). This is
intended for debugging purposes."""
class BadWriteEnablerError(Exception):
pass
-class RIControlClient(RemoteInterface):
+class RIControlClient(RemoteInterface):
def wait_for_client_connections(num_clients=int):
"""Do not return until we have connections to at least NUM_CLIENTS
storage servers.
"""
- def upload_from_file_to_uri(filename=str,
- convergence=ChoiceOf(None,
- StringConstraint(2**20))):
- """Upload a file to the grid. This accepts a filename (which must be
- absolute) that points to a file on the node's local disk. The node will
- read the contents of this file, upload it to the grid, then return the
- URI at which it was uploaded. If convergence is None then a random
- encryption key will be used, else the plaintext will be hashed, then
- that hash will be mixed together with the "convergence" string to form
- the encryption key.
- """
- return URI
+ # debug stuff
- def download_from_uri_to_file(uri=URI, filename=str):
- """Download a file from the grid, placing it on the node's local disk
- at the given filename (which must be absolute[?]). Returns the
- absolute filename where the file was written."""
+ def upload_random_data_from_file(size=int, convergence=str):
return str
- # debug stuff
+ def download_to_tempfile_and_delete(uri=str):
+ return None
def get_memory_usage():
"""Return a dict describes the amount of memory currently in use. The
def provide(provider=RIStatsProvider, nickname=str):
"""
- @param provider: a stats collector instance which should be polled
+ @param provider: a stats collector instance that should be polled
periodically by the gatherer to collect stats.
@param nickname: a name useful to identify the provided client
"""
class IValidatedThingProxy(Interface):
def start():
- """ Acquire a thing and validate it. Return a deferred which is
+ """ Acquire a thing and validate it. Return a deferred that is
eventually fired with self if the thing is valid or errbacked if it
can't be acquired or validated."""
projects / tahoe-lafs / tahoe-lafs.git / blobdiff