3 import random, math, re
4 from twisted.python import usage
6 class Args(usage.Options):
8 ["mode", "m", "alpha", "validation scheme"],
9 ["arity", "k", 2, "k (airty) for hash tree"],
11 def opt_arity(self, option):
12 self['arity'] = int(option)
13 def parseArgs(self, *args):
15 self['mode'] = args[0]
20 sizes = [random.randrange(10, 20) for i in range(10)]
26 x.title = "Example Graph"
27 #x.ext_color = [ "white", "yellow", "red", "blue", "green"]
29 #x.setLabels(["Mon", "Tue", "Wed", "Thu", "Fri"])
39 def __init__(self, mode, file_size, arity=2):
42 self.file_size = file_size
43 self.seg_size = seg_size = 1.0 * min(MAX_SEGSIZE, file_size)
44 self.num_segs = num_segs = math.ceil(file_size / seg_size)
45 self.num_blocks = num_blocks = num_segs
47 self.num_shares = num_shares = 10
48 self.shares_needed = shares_needed = 3
50 self.block_size = block_size = seg_size / shares_needed
51 self.share_size = share_size = block_size * num_blocks
53 # none of this includes the share-level hash chain yet, since that is
54 # only a function of the number of shares. All overhead numbers
55 # assume that the share-level hash chain has already been sent,
56 # including the root of the block-level hash tree.
61 self.block_tree_depth = 0
62 self.block_overhead = 0
63 self.bytes_until_some_data = 32 + share_size
64 self.share_storage_overhead = 0
65 self.share_transmission_overhead = 0
69 # each block has a 32-byte hash
70 self.block_arity = num_blocks
71 self.block_tree_depth = 1
72 self.block_overhead = 32
73 # the share has a list of hashes, one for each block
74 self.share_storage_overhead = (self.block_overhead *
76 # we can get away with not sending the hash of the share that
77 # we're sending in full, once
78 self.share_transmission_overhead = self.share_storage_overhead - 32
79 # we must get the whole list (so it can be validated) before
80 # any data can be validated
81 self.bytes_until_some_data = (self.share_transmission_overhead +
85 self.block_arity = k = arity
86 d = math.ceil(math.log(num_blocks, k))
87 self.block_tree_depth = d
89 # to make things easier, we make the pessimistic assumption that
90 # we have to store hashes for all the empty places in the tree
91 # (when the number of shares is not an exact exponent of k)
92 self.block_overhead = 32
93 # the block hashes are organized into a k-ary tree, which
94 # means storing (and eventually transmitting) more hashes. This
95 # count includes all the low-level share hashes and the root.
96 hash_nodes = (num_leaves*k - 1) / (k - 1)
97 #print "hash_depth", d
98 #print "num_leaves", num_leaves
99 #print "hash_nodes", hash_nodes
100 # the storage overhead is this
101 self.share_storage_overhead = 32 * (hash_nodes - 1)
102 # the transmission overhead is smaller: if we actually transmit
103 # every block, we don't have to transmit 1/k of the
104 # lowest-level block hashes, and we don't have to transmit the
105 # root because it was already sent with the share-level hash tree
106 self.share_transmission_overhead = 32 * (hash_nodes
109 # we must get a full sibling hash chain before we can validate
111 sibling_length = d * (k-1)
112 self.bytes_until_some_data = 32 * sibling_length + block_size
115 raise ValueError("unknown mode '%s" % mode)
117 self.storage_overhead = self.share_storage_overhead * num_shares
118 self.storage_overhead_percentage = 100.0 * self.storage_overhead / file_size
121 for k in ("mode", "file_size", "seg_size",
122 "num_segs", "num_blocks", "num_shares", "shares_needed",
123 "block_size", "share_size",
124 "block_arity", "block_tree_depth",
126 "share_storage_overhead", "share_transmission_overhead",
127 "storage_overhead", "storage_overhead_percentage",
128 "bytes_until_some_data"):
129 print k, getattr(self, k)
131 def fmt(num, trim=False):
136 s = "%.2fk" % (num / KiB)
138 s = "%.2fM" % (num / MiB)
140 s = "%.2fG" % (num / GiB)
142 s = "%.2fT" % (num / TiB)
146 s = re.sub(r'(\.0+)([kMGT#])',
147 lambda m: m.group(2),
150 s = re.sub(r'(\.0+)([kMGT#])',
151 lambda m: (" "*len(m.group(1))+m.group(2)),
161 arity = opts["arity"]
162 # 0123456789012345678901234567890123456789012345678901234567890123456
163 print "mode=%s" % mode, " arity=%d" % arity
164 print " storage storage"
165 print "Size sharesize overhead overhead k d alacrity"
167 print "------- ------- -------- -------- ---- -- --------"
168 #sizes = [2 ** i for i in range(7, 41)]
169 radix = math.sqrt(10); expstep = 2
170 radix = 2; expstep = 2
171 #radix = 10; expstep = 1
172 maxexp = int(math.ceil(math.log(1e12, radix)))+2
173 sizes = [radix ** i for i in range(2,maxexp,expstep)]
174 for file_size in sizes:
175 s = Sizes(mode, file_size, arity)
177 out += "%7s " % fmt(file_size, trim=True)
178 out += "%7s " % fmt(s.share_size)
179 out += "%8s" % fmt(s.storage_overhead)
180 out += "%10.2f " % s.storage_overhead_percentage
181 out += " %4d" % int(s.block_arity)
182 out += " %2d" % int(s.block_tree_depth)
183 out += " %8s" % fmt(s.bytes_until_some_data)
193 arity = opts["arity"]
194 g = Gnuplot.Gnuplot(debug=1)
195 g.title("overhead / alacrity tradeoffs")
196 g.xlabel("file size")
198 sizes = [2 ** i for i in range(7, 32)]
199 series = {"overhead": {}, "alacrity": {}}
200 for file_size in sizes:
201 s = Sizes(mode, file_size, arity)
202 series["overhead"][file_size] = s.storage_overhead_percentage
203 series["alacrity"][file_size] = s.bytes_until_some_data
204 g.plot([ (fs, series["overhead"][fs])
206 raw_input("press return")
209 if __name__ == '__main__':