misc/simulators/storage-overhead.py

   1 #!/usr/bin/env python
   2
   3 import sys, math
   4 from allmydata import uri, storage
   5 from allmydata.immutable import upload
   6 from allmydata.interfaces import DEFAULT_MAX_SEGMENT_SIZE
   7 from allmydata.util import mathutil
   8
   9 def roundup(size, blocksize=4096):
  10     return blocksize * mathutil.div_ceil(size, blocksize)
  11
  12
  13 class BigFakeString:
  14     def __init__(self, length):
  15         self.length = length
  16         self.fp = 0
  17     def seek(self, offset, whence=0):
  18         if whence == 0:
  19             self.fp = offset
  20         elif whence == 1:
  21             self.fp += offset
  22         elif whence == 2:
  23             self.fp = self.length - offset
  24     def tell(self):
  25         return self.fp
  26
  27 def calc(filesize, params=(3,7,10), segsize=DEFAULT_MAX_SEGMENT_SIZE):
  28     num_shares = params[2]
  29     if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD:
  30         urisize = len(uri.LiteralFileURI("A"*filesize).to_string())
  31         sharesize = 0
  32         sharespace = 0
  33     else:
  34         u = upload.FileUploader(None) # XXX changed
  35         u.set_params(params)
  36         # unfortunately, Encoder doesn't currently lend itself to answering
  37         # this question without measuring a filesize, so we have to give it a
  38         # fake one
  39         data = BigFakeString(filesize)
  40         u.set_filehandle(data)
  41         u.set_encryption_key("a"*16)
  42         sharesize, blocksize = u.setup_encoder()
  43         # how much overhead?
  44         #  0x20 bytes of offsets
  45         #  0x04 bytes of extension length
  46         #  0x1ad bytes of extension (=429)
  47         # total is 465 bytes
  48         num_segments = mathutil.div_ceil(filesize, segsize)
  49         num_share_hashes = int(math.log(mathutil.next_power_of_k(num_shares, 2),
  50                                     2)) + 1
  51         sharesize = storage.allocated_size(sharesize, num_segments,
  52                                            num_share_hashes,
  53                                            429)
  54         sharespace = num_shares * roundup(sharesize)
  55         urisize = len(uri.pack_uri(storage_index="a"*32,
  56                                    key="a"*16,
  57                                    uri_extension_hash="a"*32,
  58                                    needed_shares=params[0],
  59                                    total_shares=params[2],
  60                                    size=filesize))
  61
  62     return urisize, sharesize, sharespace
  63
  64 def main():
  65     filesize = int(sys.argv[1])
  66     urisize, sharesize, sharespace = calc(filesize)
  67     print "urisize:", urisize
  68     print "sharesize:  %10d" % sharesize
  69     print "sharespace: %10d" % sharespace
  70     print "desired expansion: %1.1f" % (1.0 * 10 / 3)
  71     print "effective expansion: %1.1f" % (1.0 * sharespace / filesize)
  72
  73 def chart():
  74     filesize = 2
  75     while filesize < 2**20:
  76         urisize, sharesize, sharespace = calc(int(filesize))
  77         expansion = 1.0 * sharespace / int(filesize)
  78         print "%d,%d,%d,%1.2f" % (int(filesize), urisize, sharespace, expansion)
  79         filesize  = filesize * 2**0.5
  80
  81 if __name__ == '__main__':
  82     if sys.argv[1] == "chart":
  83         chart()
  84     else:
  85         main()
  86