3 from zope.interface import implements
4 from twisted.python.components import registerAdapter
5 from allmydata.storage.server import si_a2b, si_b2a
6 from allmydata.util import base32, hashutil
7 from allmydata.interfaces import IURI, IDirnodeURI, IFileURI, IImmutableFileURI, \
8 IVerifierURI, IMutableFileURI, IDirectoryURI, IReadonlyDirectoryURI
10 class BadURIError(Exception):
13 # the URI shall be an ascii representation of the file. It shall contain
14 # enough information to retrieve and validate the contents. It shall be
15 # expressed in a limited character set (namely [TODO]).
17 BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits)
18 BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits)
22 NUMBER_IGNORE='(?:[0-9]+)'
24 # URIs (soon to be renamed "caps") are always allowed to come with a leading
25 # 'http://127.0.0.1:(8123|3456)/uri/' that will be ignored.
26 OPTIONALHTTPLEAD=r'(?:https?://(?:[^:/]+)(?::%s)?/uri/)?' % NUMBER_IGNORE
31 return self.to_string().__hash__()
32 def __eq__(self, them):
33 if isinstance(them, _BaseURI):
34 return self.to_string() == them.to_string()
37 def __ne__(self, them):
38 if isinstance(them, _BaseURI):
39 return self.to_string() != them.to_string()
42 def to_human_encoding(self):
43 return 'http://127.0.0.1:3456/uri/'+self.to_string()
45 def get_storage_index(self):
46 return self.storage_index
48 class CHKFileURI(_BaseURI):
49 implements(IURI, IImmutableFileURI)
51 BASE_STRING='URI:CHK:'
52 STRING_RE=re.compile('^URI:CHK:'+BASE32STR_128bits+':'+
53 BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER+
55 HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'CHK'+SEP+
56 BASE32STR_128bits+SEP+BASE32STR_256bits+SEP+NUMBER+
57 SEP+NUMBER+SEP+NUMBER+'$')
59 def __init__(self, key, uri_extension_hash, needed_shares, total_shares,
62 self.uri_extension_hash = uri_extension_hash
63 self.needed_shares = needed_shares
64 self.total_shares = total_shares
66 self.storage_index = hashutil.storage_index_hash(self.key)
67 if not len(self.storage_index) == 16: # sha256 hash truncated to 128
68 raise BadURIError("storage index must be 16 bytes long")
71 def init_from_human_encoding(cls, uri):
72 mo = cls.HUMAN_RE.search(uri)
74 raise BadURIError("%s doesn't look like a cap" % (uri,))
75 return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)),
76 int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
79 def init_from_string(cls, uri):
80 mo = cls.STRING_RE.search(uri)
82 raise BadURIError("%s doesn't look like a cap" % (uri,))
83 return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)),
84 int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
87 assert isinstance(self.needed_shares, int)
88 assert isinstance(self.total_shares, int)
89 assert isinstance(self.size, (int,long))
91 return ('URI:CHK:%s:%s:%d:%d:%d' %
92 (base32.b2a(self.key),
93 base32.b2a(self.uri_extension_hash),
98 def is_readonly(self):
100 def is_mutable(self):
102 def get_readonly(self):
108 def get_verify_cap(self):
109 return CHKFileVerifierURI(storage_index=self.storage_index,
110 uri_extension_hash=self.uri_extension_hash,
111 needed_shares=self.needed_shares,
112 total_shares=self.total_shares,
115 class CHKFileVerifierURI(_BaseURI):
116 implements(IVerifierURI)
118 BASE_STRING='URI:CHK-Verifier:'
119 STRING_RE=re.compile('^URI:CHK-Verifier:'+BASE32STR_128bits+':'+
120 BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER)
121 HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'CHK-Verifier'+SEP+
122 BASE32STR_128bits+SEP+BASE32STR_256bits+SEP+NUMBER+
123 SEP+NUMBER+SEP+NUMBER)
125 def __init__(self, storage_index, uri_extension_hash,
126 needed_shares, total_shares, size):
127 assert len(storage_index) == 16
128 self.storage_index = storage_index
129 self.uri_extension_hash = uri_extension_hash
130 self.needed_shares = needed_shares
131 self.total_shares = total_shares
135 def init_from_human_encoding(cls, uri):
136 mo = cls.HUMAN_RE.search(uri)
138 return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)),
139 int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
142 def init_from_string(cls, uri):
143 mo = cls.STRING_RE.search(uri)
144 assert mo, (uri, cls, cls.STRING_RE)
145 return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2)),
146 int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
149 assert isinstance(self.needed_shares, int)
150 assert isinstance(self.total_shares, int)
151 assert isinstance(self.size, (int,long))
153 return ('URI:CHK-Verifier:%s:%s:%d:%d:%d' %
154 (si_b2a(self.storage_index),
155 base32.b2a(self.uri_extension_hash),
161 class LiteralFileURI(_BaseURI):
162 implements(IURI, IImmutableFileURI)
164 BASE_STRING='URI:LIT:'
165 STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$')
166 HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'LIT'+SEP+base32.BASE32STR_anybytes+'$')
168 def __init__(self, data=None):
170 assert isinstance(data, str)
174 def init_from_human_encoding(cls, uri):
175 mo = cls.HUMAN_RE.search(uri)
177 return cls(base32.a2b(mo.group(1)))
180 def init_from_string(cls, uri):
181 mo = cls.STRING_RE.search(uri)
183 return cls(base32.a2b(mo.group(1)))
186 return 'URI:LIT:%s' % base32.b2a(self.data)
188 def is_readonly(self):
190 def is_mutable(self):
192 def get_readonly(self):
194 def get_storage_index(self):
197 def get_verify_cap(self):
198 # LIT files need no verification, all the data is present in the URI
202 return len(self.data)
204 class WriteableSSKFileURI(_BaseURI):
205 implements(IURI, IMutableFileURI)
207 BASE_STRING='URI:SSK:'
208 STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+
209 BASE32STR_256bits+'$')
210 HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'SSK'+SEP+
211 BASE32STR_128bits+SEP+BASE32STR_256bits+'$')
213 def __init__(self, writekey, fingerprint):
214 self.writekey = writekey
215 self.readkey = hashutil.ssk_readkey_hash(writekey)
216 self.storage_index = hashutil.ssk_storage_index_hash(self.readkey)
217 assert len(self.storage_index) == 16
218 self.fingerprint = fingerprint
221 def init_from_human_encoding(cls, uri):
222 mo = cls.HUMAN_RE.search(uri)
224 raise BadURIError("'%s' doesn't look like a cap" % (uri,))
225 return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
228 def init_from_string(cls, uri):
229 mo = cls.STRING_RE.search(uri)
231 raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
232 return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
235 assert isinstance(self.writekey, str)
236 assert isinstance(self.fingerprint, str)
237 return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey),
238 base32.b2a(self.fingerprint))
241 return "<%s %s>" % (self.__class__.__name__, self.abbrev())
244 return base32.b2a(self.writekey[:5])
246 return base32.b2a(self.storage_index)[:5]
248 def is_readonly(self):
250 def is_mutable(self):
252 def get_readonly(self):
253 return ReadonlySSKFileURI(self.readkey, self.fingerprint)
254 def get_verify_cap(self):
255 return SSKVerifierURI(self.storage_index, self.fingerprint)
257 class ReadonlySSKFileURI(_BaseURI):
258 implements(IURI, IMutableFileURI)
260 BASE_STRING='URI:SSK-RO:'
261 STRING_RE=re.compile('^URI:SSK-RO:'+BASE32STR_128bits+':'+BASE32STR_256bits+'$')
262 HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'SSK-RO'+SEP+BASE32STR_128bits+SEP+BASE32STR_256bits+'$')
264 def __init__(self, readkey, fingerprint):
265 self.readkey = readkey
266 self.storage_index = hashutil.ssk_storage_index_hash(self.readkey)
267 assert len(self.storage_index) == 16
268 self.fingerprint = fingerprint
271 def init_from_human_encoding(cls, uri):
272 mo = cls.HUMAN_RE.search(uri)
274 raise BadURIError("'%s' doesn't look like a cap" % (uri,))
275 return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
278 def init_from_string(cls, uri):
279 mo = cls.STRING_RE.search(uri)
281 raise BadURIError("'%s' doesn't look like a cap" % (uri,))
282 return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
285 assert isinstance(self.readkey, str)
286 assert isinstance(self.fingerprint, str)
287 return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey),
288 base32.b2a(self.fingerprint))
291 return "<%s %s>" % (self.__class__.__name__, self.abbrev())
294 return base32.b2a(self.readkey[:5])
296 return base32.b2a(self.storage_index)[:5]
298 def is_readonly(self):
300 def is_mutable(self):
302 def get_readonly(self):
304 def get_verify_cap(self):
305 return SSKVerifierURI(self.storage_index, self.fingerprint)
307 class SSKVerifierURI(_BaseURI):
308 implements(IVerifierURI)
310 BASE_STRING='URI:SSK-Verifier:'
311 STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'$')
312 HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'SSK-Verifier'+SEP+BASE32STR_128bits+SEP+BASE32STR_256bits+'$')
314 def __init__(self, storage_index, fingerprint):
315 assert len(storage_index) == 16
316 self.storage_index = storage_index
317 self.fingerprint = fingerprint
320 def init_from_human_encoding(cls, uri):
321 mo = cls.HUMAN_RE.search(uri)
323 return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2)))
326 def init_from_string(cls, uri):
327 mo = cls.STRING_RE.search(uri)
328 assert mo, (uri, cls)
329 return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2)))
332 assert isinstance(self.storage_index, str)
333 assert isinstance(self.fingerprint, str)
334 return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index),
335 base32.b2a(self.fingerprint))
337 class _DirectoryBaseURI(_BaseURI):
338 implements(IURI, IDirnodeURI)
339 def __init__(self, filenode_uri=None):
340 self._filenode_uri = filenode_uri
343 return "<%s %s>" % (self.__class__.__name__, self.abbrev())
346 def init_from_string(cls, uri):
347 mo = cls.BASE_STRING_RE.search(uri)
349 raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
350 bits = uri[mo.end():]
351 fn = cls.INNER_URI_CLASS.init_from_string(
352 cls.INNER_URI_CLASS.BASE_STRING+bits)
356 def init_from_human_encoding(cls, uri):
357 mo = cls.BASE_HUMAN_RE.search(uri)
359 raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
360 bits = uri[mo.end():]
361 while bits and bits[-1] == '/':
363 fn = cls.INNER_URI_CLASS.init_from_string(
364 cls.INNER_URI_CLASS.BASE_STRING+urllib.unquote(bits))
368 fnuri = self._filenode_uri.to_string()
369 mo = re.match(self.INNER_URI_CLASS.BASE_STRING, fnuri)
371 bits = fnuri[mo.end():]
372 return self.BASE_STRING+bits
375 return self._filenode_uri.to_string().split(':')[2][:5]
377 return base32.b2a(self._filenode_uri.storage_index)[:5]
379 def get_filenode_cap(self):
380 return self._filenode_uri
382 def is_mutable(self):
385 def get_verify_cap(self):
386 return DirectoryURIVerifier(self._filenode_uri.get_verify_cap())
388 def get_storage_index(self):
389 return self._filenode_uri.get_storage_index()
391 class DirectoryURI(_DirectoryBaseURI):
392 implements(IDirectoryURI)
394 BASE_STRING='URI:DIR2:'
395 BASE_STRING_RE=re.compile('^'+BASE_STRING)
396 BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2'+SEP)
397 INNER_URI_CLASS=WriteableSSKFileURI
399 def __init__(self, filenode_uri=None):
401 assert not filenode_uri.is_readonly()
402 _DirectoryBaseURI.__init__(self, filenode_uri)
404 def is_readonly(self):
407 def get_readonly(self):
408 return ReadonlyDirectoryURI(self._filenode_uri.get_readonly())
410 class ReadonlyDirectoryURI(_DirectoryBaseURI):
411 implements(IReadonlyDirectoryURI)
413 BASE_STRING='URI:DIR2-RO:'
414 BASE_STRING_RE=re.compile('^'+BASE_STRING)
415 BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-RO'+SEP)
416 INNER_URI_CLASS=ReadonlySSKFileURI
418 def __init__(self, filenode_uri=None):
420 assert filenode_uri.is_readonly()
421 _DirectoryBaseURI.__init__(self, filenode_uri)
423 def is_readonly(self):
426 def get_readonly(self):
429 class _ImmutableDirectoryBaseURI(_DirectoryBaseURI):
430 def __init__(self, filenode_uri=None):
432 assert isinstance(filenode_uri, self.INNER_URI_CLASS), filenode_uri
433 _DirectoryBaseURI.__init__(self, filenode_uri)
435 def is_mutable(self):
438 def is_readonly(self):
441 def get_readonly(self):
444 class ImmutableDirectoryURI(_ImmutableDirectoryBaseURI):
445 BASE_STRING='URI:DIR2-CHK:'
446 BASE_STRING_RE=re.compile('^'+BASE_STRING)
447 BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-CHK'+SEP)
448 INNER_URI_CLASS=CHKFileURI
449 def get_verify_cap(self):
450 vcap = self._filenode_uri.get_verify_cap()
451 return ImmutableDirectoryURIVerifier(vcap)
454 class LiteralDirectoryURI(_ImmutableDirectoryBaseURI):
455 BASE_STRING='URI:DIR2-LIT:'
456 BASE_STRING_RE=re.compile('^'+BASE_STRING)
457 BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-LIT'+SEP)
458 INNER_URI_CLASS=LiteralFileURI
459 def get_verify_cap(self):
460 # LIT caps have no verifier, since they aren't distributed
463 def wrap_dirnode_cap(filecap):
464 if isinstance(filecap, WriteableSSKFileURI):
465 return DirectoryURI(filecap)
466 if isinstance(filecap, ReadonlySSKFileURI):
467 return ReadonlyDirectoryURI(filecap)
468 if isinstance(filecap, CHKFileURI):
469 return ImmutableDirectoryURI(filecap)
470 if isinstance(filecap, LiteralFileURI):
471 return LiteralDirectoryURI(filecap)
472 assert False, "cannot wrap a dirnode around %s" % filecap.__class__
474 class DirectoryURIVerifier(_DirectoryBaseURI):
475 implements(IVerifierURI)
477 BASE_STRING='URI:DIR2-Verifier:'
478 BASE_STRING_RE=re.compile('^'+BASE_STRING)
479 BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-Verifier'+SEP)
480 INNER_URI_CLASS=SSKVerifierURI
482 def __init__(self, filenode_uri=None):
484 assert IVerifierURI.providedBy(filenode_uri)
485 self._filenode_uri = filenode_uri
487 def get_filenode_cap(self):
488 return self._filenode_uri
490 class ImmutableDirectoryURIVerifier(DirectoryURIVerifier):
491 implements(IVerifierURI)
492 BASE_STRING='URI:DIR2-CHK-Verifier:'
493 BASE_STRING_RE=re.compile('^'+BASE_STRING)
494 BASE_HUMAN_RE=re.compile('^'+OPTIONALHTTPLEAD+'URI'+SEP+'DIR2-CHK-VERIFIER'+SEP)
495 INNER_URI_CLASS=CHKFileVerifierURI
498 def __init__(self, uri):
504 if not isinstance(s, str):
505 raise TypeError("unknown URI type: %s.." % str(s)[:100])
506 elif s.startswith('URI:CHK:'):
507 return CHKFileURI.init_from_string(s)
508 elif s.startswith('URI:CHK-Verifier:'):
509 return CHKFileVerifierURI.init_from_string(s)
510 elif s.startswith('URI:LIT:'):
511 return LiteralFileURI.init_from_string(s)
512 elif s.startswith('URI:SSK:'):
513 return WriteableSSKFileURI.init_from_string(s)
514 elif s.startswith('URI:SSK-RO:'):
515 return ReadonlySSKFileURI.init_from_string(s)
516 elif s.startswith('URI:SSK-Verifier:'):
517 return SSKVerifierURI.init_from_string(s)
518 elif s.startswith('URI:DIR2:'):
519 return DirectoryURI.init_from_string(s)
520 elif s.startswith('URI:DIR2-RO:'):
521 return ReadonlyDirectoryURI.init_from_string(s)
522 elif s.startswith('URI:DIR2-Verifier:'):
523 return DirectoryURIVerifier.init_from_string(s)
524 elif s.startswith('URI:DIR2-CHK:'):
525 return ImmutableDirectoryURI.init_from_string(s)
526 elif s.startswith('URI:DIR2-LIT:'):
527 return LiteralDirectoryURI.init_from_string(s)
534 except (TypeError, AssertionError):
537 def from_string_dirnode(s):
539 assert IDirnodeURI.providedBy(u)
542 registerAdapter(from_string_dirnode, str, IDirnodeURI)
544 def from_string_filenode(s):
546 assert IFileURI.providedBy(u)
549 registerAdapter(from_string_filenode, str, IFileURI)
551 def from_string_mutable_filenode(s):
553 assert IMutableFileURI.providedBy(u)
555 registerAdapter(from_string_mutable_filenode, str, IMutableFileURI)
557 def from_string_verifier(s):
559 assert IVerifierURI.providedBy(u)
561 registerAdapter(from_string_verifier, str, IVerifierURI)
564 def pack_extension(data):
566 for k in sorted(data.keys()):
568 if isinstance(value, (int, long)):
570 assert isinstance(value, str), k
571 assert re.match(r'^[a-zA-Z_\-]+$', k)
572 pieces.append(k + ':' + hashutil.netstring(value))
573 uri_extension = ''.join(pieces)
576 def unpack_extension(data):
579 colon = data.index(':')
581 data = data[colon+1:]
583 colon = data.index(':')
584 number = data[:colon]
586 data = data[colon+1:]
588 value = data[:length]
589 assert data[length] == ','
590 data = data[length+1:]
594 # convert certain things to numbers
595 for intkey in ('size', 'segment_size', 'num_segments',
596 'needed_shares', 'total_shares'):
598 d[intkey] = int(d[intkey])
602 def unpack_extension_readable(data):
603 unpacked = unpack_extension(data)
604 unpacked["UEB_hash"] = hashutil.uri_extension_hash(data)
605 for k in sorted(unpacked.keys()):
607 unpacked[k] = base32.b2a(unpacked[k])