From: Daira Hopwood Date: Wed, 23 Sep 2015 13:33:53 +0000 (+0100) Subject: WIP (wrong). X-Git-Url: https://git.rkrishnan.org/pf/content/en/seg/bcase?a=commitdiff_plain;h=1db7925db6b83bf1bb3c3e8816f5585594284154;p=tahoe-lafs%2Ftahoe-lafs.git WIP (wrong). Signed-off-by: Daira Hopwood --- diff --git a/src/allmydata/backupdb.py b/src/allmydata/backupdb.py index 0263717a..0c60230f 100644 --- a/src/allmydata/backupdb.py +++ b/src/allmydata/backupdb.py @@ -393,8 +393,8 @@ class MagicFolderDB(BackupDB): else: return row[0] - def did_upload_file(self, filecap, relpath_u, version, mtime, ctime, size): - #print "_did_upload_file(%r, %r, %r, %r, %r, %r)" % (filecap, relpath_u, version, mtime, ctime, size) + def did_upload_file(self, filecap, relpath_u, version, pathinfo): + #print "_did_upload_file(%r, %r, %r, %r)" % (filecap, relpath_u, version, pathinfo) now = time.time() fileid = self.get_or_allocate_fileid_for_cap(filecap) try: @@ -407,35 +407,26 @@ class MagicFolderDB(BackupDB): (now, now, fileid)) try: self.cursor.execute("INSERT INTO local_files VALUES (?,?,?,?,?,?)", - (relpath_u, size, mtime, ctime, fileid, version)) + (relpath_u, pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version)) except (self.sqlite_module.IntegrityError, self.sqlite_module.OperationalError): self.cursor.execute("UPDATE local_files" " SET size=?, mtime=?, ctime=?, fileid=?, version=?" " WHERE path=?", - (size, mtime, ctime, fileid, version, relpath_u)) + (pathinfo.size, pathinfo.mtime, pathinfo.ctime, fileid, version, relpath_u)) self.connection.commit() - def is_new_file_time(self, path, relpath_u): - """recent_file_time returns true if the file is recent... - meaning its current statinfo (i.e. size, ctime, and mtime) matched the statinfo - that was previously stored in the db. + def is_new_file_time(self, pathinfo, relpath_u): """ - #print "check_file_time %s %s" % (path, relpath_u) - path = abspath_expanduser_unicode(path) - s = os.stat(path) - size = s[stat.ST_SIZE] - ctime = s[stat.ST_CTIME] - mtime = s[stat.ST_MTIME] + Returns true if the file's current pathinfo (size, mtime, and ctime) has + changed from the pathinfo previously stored in the db. + """ + #print "is_new_file_time(%s, %s)" % (pathinfo, relpath_u) c = self.cursor - c.execute("SELECT size,mtime,ctime,fileid" + c.execute("SELECT size, mtime, ctime" " FROM local_files" " WHERE path=?", (relpath_u,)) row = self.cursor.fetchone() if not row: return True - (last_size,last_mtime,last_ctime,last_fileid) = row - if (size, ctime, mtime) == (last_size, last_ctime, last_mtime): - return False - else: - return True + return (pathinfo.size, pathinfo.mtime, pathinfo.ctime) != row diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py index 17be7570..e0b0c021 100644 --- a/src/allmydata/frontends/magic_folder.py +++ b/src/allmydata/frontends/magic_folder.py @@ -118,12 +118,12 @@ class QueueMixin(HookMixin): self._stopped = False self._turn_delay = 0 - def _get_abspath(self, relpath_u): - return unicode_from_filepath(self._local_filepath.preauthChild(relpath_u)) + def _get_filepath(self, relpath_u): + return extend_filepath(self._local_filepath, relpath_u) def _get_relpath(self, filepath): print "_get_relpath(%r)" % (filepath,) - segments = filepath.asTextMode().segmentsFrom(self._local_filepath.asTextMode()) + segments = unicode_segments_from(filepath, self._local_filepath) print "segments = %r" % (segments,) return u"/".join(segments) @@ -234,15 +234,15 @@ class Uploader(QueueMixin): def _scan(self, reldir_u): self._log("scan %r" % (reldir_u,)) - abspath_u = self._get_abspath(reldir_u) + fp = self._get_filepath(reldir_u) try: - children = listdir_unicode(abspath_u) + children = listdir_filepath(fp) except EnvironmentError: raise Exception("WARNING: magic folder: permission denied on directory %s" - % quote_local_unicode_path(abspath_u)) + % quote_filepath(fp)) except FilenameEncodingError: raise Exception("WARNING: magic folder: could not list directory %s due to a filename encoding error" - % quote_local_unicode_path(abspath_u)) + % quote_filepath(fp)) d = defer.succeed(None) for child in children: @@ -271,15 +271,15 @@ class Uploader(QueueMixin): d = defer.succeed(None) def _maybe_upload(val): - abspath_u = self._get_abspath(relpath_u) - pathinfo = get_pathinfo(abspath_u) + fp = self._get_filepath(relpath_u) + pathinfo = get_pathinfo(fp) print "pending = %r, about to remove %r" % (self._pending, relpath_u) self._pending.remove(relpath_u) encoded_name_u = magicpath.path2magic(relpath_u) if not pathinfo.exists: - self._log("notified object %s disappeared (this is normal)" % quote_local_unicode_path(abspath_u)) + self._log("notified object %s disappeared (this is normal)" % quote_filepath(fp)) self._count('objects_disappeared') d2 = defer.succeed(None) if self._db.check_file_db_exists(relpath_u): @@ -305,10 +305,10 @@ class Uploader(QueueMixin): d2.addCallback(lambda x: Exception("file does not exist")) # FIXME wrong return d2 elif pathinfo.islink: - self.warn("WARNING: cannot upload symlink %s" % quote_local_unicode_path(abspath_u)) + self.warn("WARNING: cannot upload symlink %s" % quote_filepath(fp)) return None elif pathinfo.isdir: - self._notifier.watch(to_filepath(abspath_u), mask=self.mask, callbacks=[self._notify], recursive=True) + self._notifier.watch(fp, mask=self.mask, callbacks=[self._notify], recursive=True) uploadable = Data("", self._client.convergence) encoded_name_u += u"@_" upload_d = self._upload_dirnode.add_file(encoded_name_u, uploadable, metadata={"version":0}, overwrite=True) @@ -325,21 +325,19 @@ class Uploader(QueueMixin): version = self._db.get_local_file_version(relpath_u) if version is None: version = 0 - elif self._db.is_new_file_time(abspath_u, relpath_u): + elif self._db.is_new_file_time(pathinfo, relpath_u): version += 1 - uploadable = FileName(abspath_u, self._client.convergence) + uploadable = FileName(unicode_from_filepath(fp), self._client.convergence) d2 = self._upload_dirnode.add_file(encoded_name_u, uploadable, metadata={"version":version}, overwrite=True) def add_db_entry(filenode): filecap = filenode.get_uri() - # XXX maybe just pass pathinfo - self._db.did_upload_file(filecap, relpath_u, version, - pathinfo.mtime, pathinfo.ctime, pathinfo.size) + self._db.did_upload_file(filecap, relpath_u, version, pathinfo) self._count('files_uploaded') d2.addCallback(add_db_entry) return d2 else: - self.warn("WARNING: cannot process special file %s" % quote_local_unicode_path(abspath_u)) + self.warn("WARNING: cannot process special file %s" % quote_filepath(fp)) return None d.addCallback(_maybe_upload) @@ -415,8 +413,7 @@ class Downloader(QueueMixin): exists in our magic-folder db; if not then return None else check for an entry in our magic-folder db and return the version number. """ - abspath_u = self._get_abspath(relpath_u) - if not os.path.exists(abspath_u): + if not self._get_filepath(relpath_u).exists(): return None return self._db.get_local_file_version(relpath_u) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 452cdc5b..6c25403c 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -253,23 +253,40 @@ def quote_local_unicode_path(path, quotemarks=True): return quote_output(path, quotemarks=quotemarks, quote_newlines=True) -def to_filepath(path): - precondition(isinstance(path, basestring), path=path) +def quote_filepath(path, quotemarks=True): + return quote_local_unicode_path(unicode_from_filepath(path), quotemarks=quotemarks) +def _encode(s): + precondition(isinstance(s, basestring), s=s) if isinstance(path, unicode) and not use_unicode_filepath: - path = path.encode(filesystem_encoding) + return s.encode(filesystem_encoding) + else: + return s - return FilePath(path) +def _decode(s): + precondition(isinstance(s, basestring), s=s) + if isinstance(s, bytes): + return s.decode(filesystem_encoding) + else: + return s -def unicode_from_filepath(fp): - precondition(isinstance(fp, FilePath), fp=fp) +def to_filepath(path): + return FilePath(_encode(path)) - path = fp.path - if isinstance(path, bytes): - path = path.decode(filesystem_encoding) +def extend_filepath(fp, path): + return fp.preauthChild(_decode(path)) - return path +def unicode_from_filepath(fp): + precondition(isinstance(fp, FilePath), fp=fp) + return _decode(fp.path) +def unicode_segments_from(base_fp, ancestor_fp): + if hasattr(FilePath, 'asTextMode'): + return base_fp.asTextMode().segmentsFrom(ancestor_fp.asTextMode()) + else: + bpt, apt = (type(base_fp.path), type(ancestor_fp.path)) + _assert(bpt == apt, bpt=bpt, apt=apt) + return map(_decode, base_fp.segmentsFrom(ancestor_fp)) def unicode_platform(): """ @@ -317,3 +334,6 @@ def listdir_unicode(path): return os.listdir(path) else: return listdir_unicode_fallback(path) + +def listdir_filepath(fp): + return listdir_unicode(unicode_from_filepath(fp))