tw_vectors=TestAndWriteVectorsForShares,
r_vector=ReadVector,
):
- """General-purpose test-and-set operation for mutable slots. Perform
- a bunch of comparisons against the existing shares. If they all pass,
- then apply a bunch of write vectors to those shares. Then use the
- read vectors to extract data from all the shares and return the data.
+ """
+ General-purpose test-read-and-set operation for mutable slots:
+ (1) For submitted shnums, compare the test vectors against extant
+ shares, or against an empty share for shnums that do not exist.
+ (2) Use the read vectors to extract "old data" from extant shares.
+ (3) If all tests in (1) passed, then apply the write vectors
+ (possibly creating new shares).
+ (4) Return whether the tests passed, and the "old data", which does
+ not include any modifications made by the writes.
+
+ The operation does not interleave with other operations on the same
+ shareset.
This method is, um, large. The goal is to allow clients to update all
the shares associated with a mutable file in a single round trip.
Each share can have a separate test vector (i.e. a list of
comparisons to perform). If all vectors for all shares pass, then all
writes for all shares are recorded. Each comparison is a 4-tuple of
- (offset, length, operator, specimen), which effectively does a bool(
- (read(offset, length)) OPERATOR specimen ) and only performs the
- write if all these evaluate to True. Basic test-and-set uses 'eq'.
+ (offset, length, operator, specimen), which effectively does a
+ bool( (read(offset, length)) OPERATOR specimen ) and only performs
+ the write if all these evaluate to True. Basic test-and-set uses 'eq'.
Write-if-newer uses a seqnum and (offset, length, 'lt', specimen).
Write-if-same-or-newer uses 'le'.
The write vector will be applied to the given share, expanding it if
necessary. A write vector applied to a share number that did not
- exist previously will cause that share to be created.
+ exist previously will cause that share to be created. Write vectors
+ must not overlap (if they do, this will either cause an error or
+ apply them in an unspecified order). Duplicate write vectors, with
+ the same offset and data, are currently tolerated but are not
+ desirable.
In Tahoe-LAFS v1.8.3 or later (except 1.9.0a1), if you send a write
vector whose offset is beyond the end of the current data, the space
than the size of the data after applying all write vectors.
The read vector is used to extract data from all known shares,
- *before* any writes have been applied. The same vector is used for
- all shares. This captures the state that was tested by the test
- vector.
+ *before* any writes have been applied. The same read vector is used
+ for all shares. This captures the state that was tested by the test
+ vector, for extant shares.
This method returns two values: a boolean and a dict. The boolean is
True if the write vectors were applied, False if not. The dict is
Note that the nodeid here is encoded using the same base32 encoding
used by Foolscap and allmydata.util.idlib.nodeid_b2a().
-
"""
return TupleOf(bool, DictOf(int, ReadData))
repeatable way, to distribute load over many peers.
"""
-class IServer(Interface):
+class IDisplayableServer(Interface):
+ def get_nickname():
+ pass
+ def get_name():
+ pass
+ def get_longname():
+ pass
+
+class IServer(IDisplayableServer):
"""I live in the client, and represent a single server."""
def start_connecting(tub, trigger_cb):
pass
- def get_nickname():
- pass
def get_rref():
pass
def get_pushed_shares():
"""Return the number of shares that were uploaded."""
def get_sharemap():
- """Return a dict mapping share identifier to set of serverids (binary
- strings). This indicates which servers were given which shares. For
+ """Return a dict mapping share identifier to set of IServer
+ instances. This indicates which servers were given which shares. For
immutable files, the shareid is an integer (the share number, from 0
to N-1). For mutable files, it is a string of the form
'seq%d-%s-sh%d', containing the sequence number, the roothash, and
the share number."""
def get_servermap():
- """Return dict mapping server peerid to a set of share numbers."""
+ """Return dict mapping IServer instance to a set of share numbers."""
def get_timings():
"""Return dict of timing information, mapping name to seconds. All
times are floats:
files always return True."""
def needs_rebalancing():
- """Return a boolean, True if the file/dir's reliability could be
+ """Return a boolean, True if the file/dirs reliability could be
improved by moving shares to new servers. Non-distributed LIT files
always return False."""
+ # the following methods all return None for non-distributed LIT files
+
+ def get_encoding_needed():
+ """Return 'k', the number of shares required for recovery"""
+ def get_encoding_expected():
+ """Return 'N', the number of total shares generated"""
+
+ def get_share_counter_good():
+ """Return the number of distinct good shares that were found. For
+ mutable files, this counts shares for the 'best' version."""
+ def get_share_counter_wrong():
+ """For mutable files, return the number of shares for versions other
+ than the 'best' one (which is defined as being the recoverable
+ version with the highest sequence number, then the highest roothash).
+ These are either leftover shares from an older version (perhaps on a
+ server that was offline when an update occurred), shares from an
+ unrecoverable newer version, or shares from an alternate current
+ version that results from an uncoordinated write collision. For a
+ healthy file, this will equal 0. For immutable files, this will
+ always equal 0."""
- def get_data():
- """Return a dictionary that describes the state of the file/dir. LIT
- files always return an empty dictionary. Normal files and directories
- return a dictionary with the following keys (note that these use
- binary strings rather than base32-encoded ones) (also note that for
- mutable files, these counts are for the 'best' version):
-
- count-shares-good: the number of distinct good shares that were found
- count-shares-needed: 'k', the number of shares required for recovery
- count-shares-expected: 'N', the number of total shares generated
- count-good-share-hosts: the number of distinct storage servers with
- good shares. If this number is less than
- count-shares-good, then some shares are
- doubled up, increasing the correlation of
- failures. This indicates that one or more
- shares should be moved to an otherwise unused
- server, if one is available.
- count-corrupt-shares: the number of shares with integrity failures
- list-corrupt-shares: a list of 'share locators', one for each share
- that was found to be corrupt. Each share
- locator is a list of (serverid, storage_index,
- sharenum).
- count-incompatible-shares: the number of shares which are of a share
- format unknown to this checker
- list-incompatible-shares: a list of 'share locators', one for each
- share that was found to be of an unknown
- format. Each share locator is a list of
- (serverid, storage_index, sharenum).
- servers-responding: list of (binary) storage server identifiers,
- one for each server which responded to the share
- query (even if they said they didn't have
- shares, and even if they said they did have
- shares but then didn't send them when asked, or
- dropped the connection, or returned a Failure,
- and even if they said they did have shares and
- sent incorrect ones when asked)
- sharemap: dict mapping share identifier to list of serverids
- (binary strings). This indicates which servers are holding
- which shares. For immutable files, the shareid is an
- integer (the share number, from 0 to N-1). For mutable
- files, it is a string of the form 'seq%d-%s-sh%d',
- containing the sequence number, the roothash, and the
- share number.
-
- The following keys are most relevant for mutable files, but immutable
- files will provide sensible values too::
-
- count-wrong-shares: the number of shares for versions other than the
- 'best' one (which is defined as being the
- recoverable version with the highest sequence
- number, then the highest roothash). These are
- either leftover shares from an older version
- (perhaps on a server that was offline when an
- update occurred), shares from an unrecoverable
- newer version, or shares from an alternate
- current version that results from an
- uncoordinated write collision. For a healthy
- file, this will equal 0.
-
- count-recoverable-versions: the number of recoverable versions of
- the file. For a healthy file, this will
- equal 1.
-
- count-unrecoverable-versions: the number of unrecoverable versions
- of the file. For a healthy file, this
- will be 0.
+ def get_corrupt_shares():
+ """Return a list of 'share locators', one for each share that was
+ found to be corrupt (integrity failure). Each share locator is a list
+ of (IServer, storage_index, sharenum)."""
+
+ def get_incompatible_shares():
+ """Return a list of 'share locators', one for each share that was
+ found to be of an unknown format. Each share locator is a list of
+ (IServer, storage_index, sharenum)."""
+
+ def get_servers_responding():
+ """Return a list of IServer objects, one for each server which
+ responded to the share query (even if they said they didn't have
+ shares, and even if they said they did have shares but then didn't
+ send them when asked, or dropped the connection, or returned a
+ Failure, and even if they said they did have shares and sent
+ incorrect ones when asked)"""
+
+ def get_host_counter_good_shares():
+ """Return the number of distinct storage servers with good shares. If
+ this number is less than get_share_counters()[good], then some shares
+ are doubled up, increasing the correlation of failures. This
+ indicates that one or more shares should be moved to an otherwise
+ unused server, if one is available.
+ """
+
+ def get_version_counter_recoverable():
+ """Return the number of recoverable versions of the file. For a
+ healthy file, this will equal 1."""
+
+ def get_version_counter_unrecoverable():
+ """Return the number of unrecoverable versions of the file. For a
+ healthy file, this will be 0."""
- """
+ def get_sharemap():
+ """Return a dict mapping share identifier to list of IServer objects.
+ This indicates which servers are holding which shares. For immutable
+ files, the shareid is an integer (the share number, from 0 to N-1).
+ For mutable files, it is a string of the form 'seq%d-%s-sh%d',
+ containing the sequence number, the roothash, and the share number."""
def get_summary():
"""Return a string with a brief (one-line) summary of the results."""
"""
def get_corrupt_shares():
- """Return a set of (serverid, storage_index, sharenum) for all shares
- that were found to be corrupt. Both serverid and storage_index are
- binary.
- """
+ """Return a set of (IServer, storage_index, sharenum) for all shares
+ that were found to be corrupt. storage_index is binary."""
def get_all_results():
"""Return a dictionary mapping pathname (a tuple of strings, ready to
be slash-joined) to an ICheckResults instance, one for each object
IDirectoryNode.deep_stats()."""
def get_corrupt_shares():
- """Return a set of (serverid, storage_index, sharenum) for all shares
- that were found to be corrupt before any repair was attempted. Both
- serverid and storage_index are binary.
+ """Return a set of (IServer, storage_index, sharenum) for all shares
+ that were found to be corrupt before any repair was attempted.
+ storage_index is binary.
"""
def get_remaining_corrupt_shares():
- """Return a set of (serverid, storage_index, sharenum) for all shares
- that were found to be corrupt after any repair was completed. Both
- serverid and storage_index are binary. These are shares that need
- manual inspection and probably deletion.
+ """Return a set of (IServer, storage_index, sharenum) for all shares
+ that were found to be corrupt after any repair was completed.
+ storage_index is binary. These are shares that need manual inspection
+ and probably deletion.
"""
def get_all_results():
"""Return a dictionary mapping pathname (a tuple of strings, ready to
projects / tahoe-lafs / tahoe-lafs.git / blobdiff