from allmydata.storage.lease import LeaseInfo
from allmydata.storage.mutable import MutableShareFile, EmptyShare, \
create_mutable_sharefile
+from allmydata.mutable.layout import MAX_MUTABLE_SHARE_SIZE
from allmydata.storage.immutable import ShareFile, BucketWriter, BucketReader
from allmydata.storage.crawler import BucketCountingCrawler
from allmydata.storage.expirer import LeaseCheckingCrawler
def __repr__(self):
return "<StorageServer %s>" % (idlib.shortnodeid_b2a(self.my_nodeid),)
+ def have_shares(self):
+ # quick test to decide if we need to commit to an implicit
+ # permutation-seed or if we should use a new one
+ return bool(set(os.listdir(self.sharedir)) - set(["incoming"]))
+
def add_bucket_counter(self):
statefile = os.path.join(self.storedir, "bucket_counter.state")
self.bucket_counter = BucketCountingCrawler(self, statefile)
def get_latencies(self):
"""Return a dict, indexed by category, that contains a dict of
- latency numbers for each category. Each dict will contain the
+ latency numbers for each category. If there are sufficient samples
+ for unambiguous interpretation, each dict will contain the
following keys: mean, 01_0_percentile, 10_0_percentile,
50_0_percentile (median), 90_0_percentile, 95_0_percentile,
- 99_0_percentile, 99_9_percentile. If no samples have been collected
- for the given category, then that category name will not be present
- in the return value."""
+ 99_0_percentile, 99_9_percentile. If there are insufficient
+ samples for a given percentile to be interpreted unambiguously
+ that percentile will be reported as None. If no samples have been
+ collected for the given category, then that category name will
+ not be present in the return value. """
# note that Amazon's Dynamo paper says they use 99.9% percentile.
output = {}
for category in self.latencies:
continue
stats = {}
samples = self.latencies[category][:]
- samples.sort()
count = len(samples)
- stats["mean"] = sum(samples) / count
- stats["01_0_percentile"] = samples[int(0.01 * count)]
- stats["10_0_percentile"] = samples[int(0.1 * count)]
- stats["50_0_percentile"] = samples[int(0.5 * count)]
- stats["90_0_percentile"] = samples[int(0.9 * count)]
- stats["95_0_percentile"] = samples[int(0.95 * count)]
- stats["99_0_percentile"] = samples[int(0.99 * count)]
- stats["99_9_percentile"] = samples[int(0.999 * count)]
+ stats["samplesize"] = count
+ samples.sort()
+ if count > 1:
+ stats["mean"] = sum(samples) / count
+ else:
+ stats["mean"] = None
+
+ orderstatlist = [(0.01, "01_0_percentile", 100), (0.1, "10_0_percentile", 10),\
+ (0.50, "50_0_percentile", 10), (0.90, "90_0_percentile", 10),\
+ (0.95, "95_0_percentile", 20), (0.99, "99_0_percentile", 100),\
+ (0.999, "99_9_percentile", 1000)]
+
+ for percentile, percentilestring, minnumtoobserve in orderstatlist:
+ if count >= minnumtoobserve:
+ stats[percentilestring] = samples[int(percentile*count)]
+ else:
+ stats[percentilestring] = None
+
output[category] = stats
return output
stats['storage_server.latencies.%s.%s' % (category, name)] = v
try:
- disk = fileutil.get_disk_stats(self.storedir, self.reserved_space)
+ disk = fileutil.get_disk_stats(self.sharedir, self.reserved_space)
writeable = disk['avail'] > 0
# spacetime predictors should use disk_avail / (d(disk_used)/dt)
if self.readonly_storage:
return 0
- return fileutil.get_available_space(self.storedir, self.reserved_space)
+ return fileutil.get_available_space(self.sharedir, self.reserved_space)
def allocated_size(self):
space = 0
version = { "http://allmydata.org/tahoe/protocols/storage/v1" :
{ "maximum-immutable-share-size": remaining_space,
+ "maximum-mutable-share-size": MAX_MUTABLE_SHARE_SIZE,
+ "available-space": remaining_space,
"tolerates-immutable-read-overrun": True,
"delete-mutable-shares-with-zero-length-writev": True,
+ "fills-holes-with-zero-bytes": True,
+ "prevents-read-past-end-of-share-data": True,
},
"application-version": str(allmydata.__full_version__),
}
if not found_buckets:
raise IndexError("no such lease to renew")
- def remote_cancel_lease(self, storage_index, cancel_secret):
- start = time.time()
- self.count("cancel")
-
- total_space_freed = 0
- found_buckets = False
- for sf in self._iter_share_files(storage_index):
- # note: if we can't find a lease on one share, we won't bother
- # looking in the others. Unless something broke internally
- # (perhaps we ran out of disk space while adding a lease), the
- # leases on all shares will be identical.
- found_buckets = True
- # this raises IndexError if the lease wasn't present XXXX
- total_space_freed += sf.cancel_lease(cancel_secret)
-
- if found_buckets:
- storagedir = os.path.join(self.sharedir,
- storage_index_to_dir(storage_index))
- if not os.listdir(storagedir):
- os.rmdir(storagedir)
-
- if self.stats_provider:
- self.stats_provider.count('storage_server.bytes_freed',
- total_space_freed)
- self.add_latency("cancel", time.time() - start)
- if not found_buckets:
- raise IndexError("no such storage index")
-
def bucket_writer_closed(self, bw, consumed_size):
if self.stats_provider:
self.stats_provider.count('storage_server.bytes_added', consumed_size)
share_type=share_type, si=si_s, shnum=shnum, reason=reason,
level=log.SCARY, umid="SGx2fA")
return None
-
projects / tahoe-lafs / tahoe-lafs.git / blobdiff