Return-Path: X-Original-To: mbp@sourcefrog.net Delivered-To: mbp@ozlabs.org X-Greylist: delayed 1826 seconds by postgrey-1.21 at ozlabs; Sun, 15 May 2005 06:59:11 EST Received: from upstroke.tntech.dk (cpe.atm2-0-1041078.0x503eaf62.odnxx4.customer.tele.dk [80.62.175.98]) by ozlabs.org (Postfix) with ESMTP id B968E679EA for ; Sun, 15 May 2005 06:59:11 +1000 (EST) Received: by upstroke.tntech.dk (Postfix, from userid 1001) id 63F83542FF; Sat, 14 May 2005 22:28:37 +0200 (CEST) To: Martin Pool Subject: [PATCH] symlink support patch From: Erik Toubro Nielsen Date: Sat, 14 May 2005 22:28:37 +0200 Message-ID: <86u0l57dsa.fsf@upstroke.tntech.dk> User-Agent: Gnus/5.1006 (Gnus v5.10.6) XEmacs/21.4 (Security Through Obscurity, linux) MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" X-Spam-Checker-Version: SpamAssassin 3.0.3 (2005-04-27) on ozlabs.org X-Spam-Level: X-Spam-Status: No, score=-1.4 required=3.2 tests=BAYES_00,NO_MORE_FUNN, RCVD_IN_BLARS_RBL autolearn=no version=3.0.3 Content-Length: 19688 Lines: 604 --=-=-= I'm not sending this to the list as it is pretty large. Let me know if its usefull and if I should rework anything. Erik Overview: (Bugfix: in class TreeDelta I've moved kind= one level up, since kind is also used in the else part) Since both the InventoryEntry and stat cache is changed, perhaps the branch format number should be increased? Added test cases for symlinks to testbzr Cannot use realpath since it expands path/L to path/LinkTarget Cannot use exists, use new osutils.lexists I'm overloading the text_modified to signify that a link target is changed. Perhaps text_modified should be renamed content_modified? InventoryEntry has a new member "symlink_target", The stat cache entry has been extended to contain the symlink target and the st_mode. I try to ignore an old format cache file. --=-=-= Content-Type: text/x-patch Content-Disposition: inline; filename=symlinksupport.patch *** modified file 'bzrlib/add.py' --- bzrlib/add.py +++ bzrlib/add.py @@ -38,7 +38,7 @@ for f in file_list: kind = bzrlib.osutils.file_kind(f) - if kind != 'file' and kind != 'directory': + if kind != 'file' and kind != 'directory' and kind != 'symlink': if f not in user_list: print "Skipping %s (can't add file of kind '%s')" % (f, kind) continue @@ -56,7 +56,7 @@ kind = bzrlib.osutils.file_kind(f) - if kind != 'file' and kind != 'directory': + if kind != 'file' and kind != 'directory' and kind != 'symlink': bailout("can't add file '%s' of kind %r" % (f, kind)) versioned = (inv.path2id(rf) != None) *** modified file 'bzrlib/branch.py' --- bzrlib/branch.py +++ bzrlib/branch.py @@ -58,11 +58,9 @@ run into the root.""" if f == None: f = os.getcwd() - elif hasattr(os.path, 'realpath'): - f = os.path.realpath(f) else: - f = os.path.abspath(f) - if not os.path.exists(f): + f = bzrlib.osutils.normalizepath(f) + if not bzrlib.osutils.lexists(f): raise BzrError('%r does not exist' % f) @@ -189,7 +187,7 @@ """Return path relative to this branch of something inside it. Raises an error if path is not in this branch.""" - rp = os.path.realpath(path) + rp = bzrlib.osutils.normalizepath(path) # FIXME: windows if not rp.startswith(self.base): bailout("path %r is not within branch %r" % (rp, self.base)) @@ -531,7 +529,9 @@ file_id = entry.file_id mutter('commit prep file %s, id %r ' % (p, file_id)) - if not os.path.exists(p): + # it should be enough to use os.lexists instead of exists + # but lexists in an 2.4 function + if not bzrlib.osutils.lexists(p): mutter(" file is missing, removing from inventory") if verbose: show_status('D', entry.kind, quotefn(path)) @@ -554,6 +554,10 @@ if entry.kind == 'directory': if not isdir(p): bailout("%s is entered as directory but not a directory" % quotefn(p)) + elif entry.kind == 'symlink': + if not os.path.islink(p): + bailout("%s is entered as symbolic link but is not a symbolic link" % quotefn(p)) + entry.read_symlink_target(p) elif entry.kind == 'file': if not isfile(p): bailout("%s is entered as file but is not a file" % quotefn(p)) *** modified file 'bzrlib/diff.py' --- bzrlib/diff.py +++ bzrlib/diff.py @@ -66,6 +66,11 @@ print >>to_file, "\\ No newline at end of file" print >>to_file +def _diff_symlink(old_tree, new_tree, file_id): + t1 = old_tree.get_symlink_target(file_id) + t2 = new_tree.get_symlink_target(file_id) + print '*** *** target changed %r => %r' % (t1, t2) + def show_diff(b, revision, specific_files): import sys @@ -112,12 +117,15 @@ for old_path, new_path, file_id, kind, text_modified in delta.renamed: print '*** renamed %s %r => %r' % (kind, old_path, new_path) - if text_modified: + if kind == 'file' and text_modified: _diff_one(old_tree.get_file(file_id).readlines(), new_tree.get_file(file_id).readlines(), sys.stdout, fromfile=old_label + old_path, tofile=new_label + new_path) + + elif kind == 'symlink' and text_modified: + _diff_symlink(old_tree, new_tree, file_id) for path, file_id, kind in delta.modified: print '*** modified %s %r' % (kind, path) @@ -128,6 +136,8 @@ fromfile=old_label + path, tofile=new_label + path) + elif kind == 'symlink': + _diff_symlink(old_tree, new_tree, file_id) class TreeDelta: @@ -149,7 +159,9 @@ Each id is listed only once. Files that are both modified and renamed are listed only in - renamed, with the text_modified flag true. + renamed, with the text_modified flag true. The text_modified + applies either to the the content of the file or the target of the + symbolic link, depending of the kind of file. The lists are normally sorted when the delta is created. """ @@ -224,8 +236,8 @@ specific_files = ImmutableSet(specific_files) for file_id in old_tree: + kind = old_inv.get_file_kind(file_id) if file_id in new_tree: - kind = old_inv.get_file_kind(file_id) assert kind == new_inv.get_file_kind(file_id) assert kind in ('file', 'directory', 'symlink', 'root_directory'), \ @@ -246,6 +258,14 @@ old_sha1 = old_tree.get_file_sha1(file_id) new_sha1 = new_tree.get_file_sha1(file_id) text_modified = (old_sha1 != new_sha1) + elif kind == 'symlink': + t1 = old_tree.get_symlink_target(file_id) + t2 = new_tree.get_symlink_target(file_id) + if t1 != t2: + mutter(" symlink target changed") + text_modified = True + else: + text_modified = False else: ## mutter("no text to check for %r %r" % (file_id, kind)) text_modified = False *** modified file 'bzrlib/inventory.py' --- bzrlib/inventory.py +++ bzrlib/inventory.py @@ -125,14 +125,22 @@ self.kind = kind self.text_id = text_id self.parent_id = parent_id + self.symlink_target = None if kind == 'directory': self.children = {} elif kind == 'file': pass + elif kind == 'symlink': + pass else: raise BzrError("unhandled entry kind %r" % kind) - + def read_symlink_target(self, path): + if self.kind == 'symlink': + try: + self.symlink_target = os.readlink(path) + except OSError,e: + raise BzrError("os.readlink error, %s" % e) def sorted_children(self): l = self.children.items() @@ -145,6 +153,7 @@ self.parent_id, text_id=self.text_id) other.text_sha1 = self.text_sha1 other.text_size = self.text_size + other.symlink_target = self.symlink_target return other @@ -168,7 +177,7 @@ if self.text_size != None: e.set('text_size', '%d' % self.text_size) - for f in ['text_id', 'text_sha1']: + for f in ['text_id', 'text_sha1', 'symlink_target']: v = getattr(self, f) if v != None: e.set(f, v) @@ -198,6 +207,7 @@ self = cls(elt.get('file_id'), elt.get('name'), elt.get('kind'), parent_id) self.text_id = elt.get('text_id') self.text_sha1 = elt.get('text_sha1') + self.symlink_target = elt.get('symlink_target') ## mutter("read inventoryentry: %r" % (elt.attrib)) *** modified file 'bzrlib/osutils.py' --- bzrlib/osutils.py +++ bzrlib/osutils.py @@ -58,7 +58,30 @@ else: raise BzrError("can't handle file kind with mode %o of %r" % (mode, f)) - +def lexists(f): + try: + if hasattr(os, 'lstat'): + os.lstat(f) + else: + os.stat(f) + return True + except OSError,e: + if e.errno == errno.ENOENT: + return False; + else: + raise BzrError("lstat/stat of (%r): %r" % (f, e)) + +def normalizepath(f): + if hasattr(os.path, 'realpath'): + F = os.path.realpath + else: + F = os.path.abspath + [p,e] = os.path.split(f) + if e == "" or e == "." or e == "..": + return F(f) + else: + return os.path.join(F(p), e) + def isdir(f): """True if f is an accessible directory.""" *** modified file 'bzrlib/statcache.py' --- bzrlib/statcache.py +++ bzrlib/statcache.py @@ -53,7 +53,7 @@ to use a tdb instead. The cache is represented as a map from file_id to a tuple of (file_id, -sha1, path, size, mtime, ctime, ino, dev). +sha1, path, symlink target, size, mtime, ctime, ino, dev, mode). """ @@ -62,11 +62,13 @@ FP_CTIME = 2 FP_INO = 3 FP_DEV = 4 - +FP_ST_MODE=5 SC_FILE_ID = 0 SC_SHA1 = 1 - +SC_SYMLINK_TARGET = 3 + +CACHE_ENTRY_SIZE = 10 def fingerprint(abspath): try: @@ -79,7 +81,7 @@ return None return (fs.st_size, fs.st_mtime, - fs.st_ctime, fs.st_ino, fs.st_dev) + fs.st_ctime, fs.st_ino, fs.st_dev, fs.st_mode) def _write_cache(basedir, entry_iter, dangerfiles): @@ -93,7 +95,9 @@ continue outf.write(entry[0] + ' ' + entry[1] + ' ') outf.write(b2a_qp(entry[2], True)) - outf.write(' %d %d %d %d %d\n' % entry[3:]) + outf.write(' ') + outf.write(b2a_qp(entry[3], True)) # symlink_target + outf.write(' %d %d %d %d %d %d\n' % entry[4:]) outf.commit() finally: @@ -114,10 +118,13 @@ for l in cachefile: f = l.split(' ') + if len(f) != CACHE_ENTRY_SIZE: + mutter("cache is in old format, must recreate it") + return {} file_id = f[0] if file_id in cache: raise BzrError("duplicated file_id in cache: {%s}" % file_id) - cache[file_id] = (f[0], f[1], a2b_qp(f[2])) + tuple([long(x) for x in f[3:]]) + cache[file_id] = (f[0], f[1], a2b_qp(f[2]), a2b_qp(f[3])) + tuple([long(x) for x in f[4:]]) return cache @@ -125,7 +132,7 @@ def _files_from_inventory(inv): for path, ie in inv.iter_entries(): - if ie.kind != 'file': + if ie.kind != 'file' and ie.kind != 'symlink': continue yield ie.file_id, path @@ -190,17 +197,24 @@ if (fp[FP_MTIME] >= now) or (fp[FP_CTIME] >= now): dangerfiles.add(file_id) - if cacheentry and (cacheentry[3:] == fp): + if cacheentry and (cacheentry[4:] == fp): continue # all stat fields unchanged hardcheck += 1 - dig = sha.new(file(abspath, 'rb').read()).hexdigest() - + mode = fp[FP_ST_MODE] + if stat.S_ISREG(mode): + link_target = '-' # can be anything, but must be non-empty + dig = sha.new(file(abspath, 'rb').read()).hexdigest() + elif stat.S_ISLNK(mode): + link_target = os.readlink(abspath) + dig = sha.new(link_target).hexdigest() + else: + raise BzrError("file %r: unknown file stat mode: %o"%(abspath,mode)) if cacheentry == None or dig != cacheentry[1]: # if there was no previous entry for this file, or if the # SHA has changed, then update the cache - cacheentry = (file_id, dig, path) + fp + cacheentry = (file_id, dig, path, link_target) + fp cache[file_id] = cacheentry change_cnt += 1 *** modified file 'bzrlib/tree.py' --- bzrlib/tree.py +++ bzrlib/tree.py @@ -125,6 +125,11 @@ os.mkdir(fullpath) elif kind == 'file': pumpfile(self.get_file(ie.file_id), file(fullpath, 'wb')) + elif kind == 'symlink': + try: + os.symlink(ie.symlink_target, fullpath) + except OSError,e: + bailout("Failed to create symlink %r -> %r, error: %s" % (fullpath, ie.symlink_target, e)) else: bailout("don't know how to export {%s} of kind %r" % (ie.file_id, kind)) mutter(" export {%s} kind %s to %s" % (ie.file_id, kind, fullpath)) @@ -167,6 +172,9 @@ for path, entry in self.inventory.iter_entries(): yield path, 'V', entry.kind, entry.file_id + def get_symlink_target(self, file_id): + ie = self._inventory[file_id] + return ie.symlink_target; class EmptyTree(Tree): def __init__(self): @@ -179,7 +187,8 @@ if False: # just to make it a generator yield None - + def get_symlink_target(self, file_id): + return None ###################################################################### # diff @@ -245,3 +254,7 @@ if old_name != new_name: yield (old_name, new_name) + + def get_symlink_target(self, file_id): + ie = self._inventory[file_id] + return ie.symlink_target *** modified file 'bzrlib/workingtree.py' --- bzrlib/workingtree.py +++ bzrlib/workingtree.py @@ -53,7 +53,7 @@ ie = inv[file_id] if ie.kind == 'file': if ((file_id in self._statcache) - or (os.path.exists(self.abspath(inv.id2path(file_id))))): + or (bzrlib.osutils.lexists(self.abspath(inv.id2path(file_id))))): yield file_id @@ -66,7 +66,7 @@ return os.path.join(self.basedir, filename) def has_filename(self, filename): - return os.path.exists(self.abspath(filename)) + return bzrlib.osutils.lexists(self.abspath(filename)) def get_file(self, file_id): return self.get_file_byname(self.id2path(file_id)) @@ -86,7 +86,7 @@ self._update_statcache() if file_id in self._statcache: return True - return os.path.exists(self.abspath(self.id2path(file_id))) + return bzrlib.osutils.lexists(self.abspath(self.id2path(file_id))) __contains__ = has_id @@ -108,6 +108,12 @@ return self._statcache[file_id][statcache.SC_SHA1] + def get_symlink_target(self, file_id): + import statcache + self._update_statcache() + target = self._statcache[file_id][statcache.SC_SYMLINK_TARGET] + return target + def file_class(self, filename): if self.path2id(filename): return 'V' *** modified file 'testbzr' --- testbzr +++ testbzr @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # Copyright (C) 2005 Canonical Ltd @@ -113,6 +113,17 @@ logfile.write(' at %s:%d\n' % stack[:2]) +def has_symlinks(): + import os; + if hasattr(os, 'symlink'): + return True + else: + return False + +def listdir_sorted(dir): + L = os.listdir(dir) + L.sort() + return L # prepare an empty scratch directory if os.path.exists(TESTDIR): @@ -320,8 +331,105 @@ runcmd('bzr ignore *.blah') assert backtick('bzr unknowns') == '' assert file('.bzrignore', 'rt').read() == '*.blah\n' - - + cd("..") + + if has_symlinks(): + progress("symlinks") + mkdir('symlinks') + cd('symlinks') + runcmd('bzr init') + os.symlink("NOWHERE1", "link1") + runcmd('bzr add link1') + assert backtick('bzr unknowns') == '' + runcmd(['bzr', 'commit', '-m', '1: added symlink link1']) + + mkdir('d1') + runcmd('bzr add d1') + assert backtick('bzr unknowns') == '' + os.symlink("NOWHERE2", "d1/link2") + assert backtick('bzr unknowns') == 'd1/link2\n' + # is d1/link2 found when adding d1 + runcmd('bzr add d1') + assert backtick('bzr unknowns') == '' + os.symlink("NOWHERE3", "d1/link3") + assert backtick('bzr unknowns') == 'd1/link3\n' + runcmd(['bzr', 'commit', '-m', '2: added dir, symlink']) + + runcmd('bzr rename d1 d2') + runcmd('bzr move d2/link2 .') + runcmd('bzr move link1 d2') + assert os.readlink("./link2") == "NOWHERE2" + assert os.readlink("d2/link1") == "NOWHERE1" + runcmd('bzr add d2/link3') + runcmd('bzr diff') + runcmd(['bzr', 'commit', '-m', '3: rename of dir, move symlinks, add link3']) + + os.unlink("link2") + os.symlink("TARGET 2", "link2") + os.unlink("d2/link1") + os.symlink("TARGET 1", "d2/link1") + runcmd('bzr diff') + assert backtick("bzr relpath d2/link1") == "d2/link1\n" + runcmd(['bzr', 'commit', '-m', '4: retarget of two links']) + + runcmd('bzr remove d2/link1') + assert backtick('bzr unknowns') == 'd2/link1\n' + runcmd(['bzr', 'commit', '-m', '5: remove d2/link1']) + + os.mkdir("d1") + runcmd('bzr add d1') + runcmd('bzr rename d2/link3 d1/link3new') + assert backtick('bzr unknowns') == 'd2/link1\n' + runcmd(['bzr', 'commit', '-m', '6: remove d2/link1, move/rename link3']) + + runcmd(['bzr', 'check']) + + runcmd(['bzr', 'export', '-r', '1', 'exp1.tmp']) + cd("exp1.tmp") + assert listdir_sorted(".") == [ "link1" ] + assert os.readlink("link1") == "NOWHERE1" + cd("..") + + runcmd(['bzr', 'export', '-r', '2', 'exp2.tmp']) + cd("exp2.tmp") + assert listdir_sorted(".") == [ "d1", "link1" ] + cd("..") + + runcmd(['bzr', 'export', '-r', '3', 'exp3.tmp']) + cd("exp3.tmp") + assert listdir_sorted(".") == [ "d2", "link2" ] + assert listdir_sorted("d2") == [ "link1", "link3" ] + assert os.readlink("d2/link1") == "NOWHERE1" + assert os.readlink("link2") == "NOWHERE2" + cd("..") + + runcmd(['bzr', 'export', '-r', '4', 'exp4.tmp']) + cd("exp4.tmp") + assert listdir_sorted(".") == [ "d2", "link2" ] + assert os.readlink("d2/link1") == "TARGET 1" + assert os.readlink("link2") == "TARGET 2" + assert listdir_sorted("d2") == [ "link1", "link3" ] + cd("..") + + runcmd(['bzr', 'export', '-r', '5', 'exp5.tmp']) + cd("exp5.tmp") + assert listdir_sorted(".") == [ "d2", "link2" ] + assert os.path.islink("link2") + assert listdir_sorted("d2")== [ "link3" ] + cd("..") + + runcmd(['bzr', 'export', '-r', '6', 'exp6.tmp']) + cd("exp6.tmp") + assert listdir_sorted(".") == [ "d1", "d2", "link2" ] + assert listdir_sorted("d1") == [ "link3new" ] + assert listdir_sorted("d2") == [] + assert os.readlink("d1/link3new") == "NOWHERE3" + cd("..") + + cd("..") + else: + progress("skipping symlink tests") + progress("all tests passed!") except Exception, e: sys.stderr.write('*' * 50 + '\n' --=-=-=--