22
22
"""Weave - storage of related text file versions"""
24
# before intset (r923) 2000 versions in 41.5s
25
# with intset (r926) 2000 versions in 93s !!!
26
# better to just use plain sets.
28
# making _extract build and return a list, rather than being a generator
31
# with python -O, r923 does 2000 versions in 36.87s
33
# with optimizations to avoid mutating lists - 35.75! I guess copying
34
# all the elements every time costs more than the small manipulations.
35
# a surprisingly small change.
37
# r931, which avoids using a generator for extract, does 36.98s
39
# with memoized inclusions, takes 41.49s; not very good
41
# with slots, takes 37.35s; without takes 39.16, a bit surprising
43
# with the delta calculation mixed in with the add method, rather than
44
# separated, takes 36.78s
46
# with delta folded in and mutation of the list, 36.13s
48
# with all this and simplification of add code, 33s
51
# TODO: Perhaps have copy method for Weave instances?
53
25
# XXX: If we do weaves this way, will a merge still behave the same
54
26
# way if it's done in a different order? That's a pretty desirable
74
44
# TODO: Parallel-extract that passes back each line along with a
75
45
# description of which revisions include it. Nice for checking all
81
class WeaveError(Exception):
82
"""Exception in processing weave"""
85
class WeaveFormatError(WeaveError):
86
"""Weave invariant violated"""
46
# shas or calculating stats in parallel.
48
# TODO: Using a single _extract routine and then processing the output
49
# is probably inefficient. It's simple enough that we can afford to
50
# have slight specializations for different ways its used: annotate,
51
# basis for add, get, etc.
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of
65
# something in the list.) We should eliminate them. This can
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
71
from difflib import SequenceMatcher
73
from bzrlib.trace import mutter
74
from bzrlib.errors import WeaveError, WeaveFormatError, WeaveParentMismatch, \
75
WeaveRevisionNotPresent, WeaveRevisionAlreadyPresent
76
from bzrlib.tsort import topo_sort
89
79
class Weave(object):
90
80
"""weave - versioned text file storage.
160
152
each version; the parent's parents are implied.
163
List of hex SHA-1 of each version, or None if not recorded.
155
List of hex SHA-1 of each version.
158
List of symbolic names for each version. Each should be unique.
161
For each name, the version number.
164
Descriptive name of this weave; typically the filename if known.
166
__slots__ = ['_weave', '_parents', '_sha1s']
168
__slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map',
171
def __init__(self, weave_name=None):
170
173
self._parents = []
177
self._weave_name = weave_name
180
return "Weave(%r)" % self._weave_name
184
"""Return a deep copy of self.
186
The copy can be modified without affecting the original weave."""
188
other._weave = self._weave[:]
189
other._parents = self._parents[:]
190
other._sha1s = self._sha1s[:]
191
other._names = self._names[:]
192
other._name_map = self._name_map.copy()
193
other._weave_name = self._weave_name
174
196
def __eq__(self, other):
175
197
if not isinstance(other, Weave):
177
199
return self._parents == other._parents \
178
and self._weave == other._weave
200
and self._weave == other._weave \
201
and self._sha1s == other._sha1s
181
204
def __ne__(self, other):
182
205
return not self.__eq__(other)
185
def add(self, parents, text):
207
def __contains__(self, name):
208
return self._name_map.has_key(name)
210
def maybe_lookup(self, name_or_index):
211
"""Convert possible symbolic name to index, or pass through indexes."""
212
if isinstance(name_or_index, (int, long)):
215
return self.lookup(name_or_index)
218
def lookup(self, name):
219
"""Convert symbolic version name to index."""
221
return self._name_map[name]
223
raise WeaveRevisionNotPresent(name, self)
226
return self._names[:]
228
def iter_names(self):
229
"""Yield a list of all names in this weave."""
230
return iter(self._names)
232
def idx_to_name(self, version):
233
return self._names[version]
235
def _check_repeated_add(self, name, parents, text, sha1):
236
"""Check that a duplicated add is OK.
238
If it is, return the (old) index; otherwise raise an exception.
240
idx = self.lookup(name)
241
if sorted(self._parents[idx]) != sorted(parents) \
242
or sha1 != self._sha1s[idx]:
243
raise WeaveRevisionAlreadyPresent(name, self)
246
def add(self, name, parents, text, sha1=None):
186
247
"""Add a single text on top of the weave.
188
249
Returns the index number of the newly added version.
252
Symbolic name for this version.
253
(Typically the revision-id of the revision that added it.)
191
256
List or set of direct parent version numbers.
194
Sequence of lines to be added in the new version."""
259
Sequence of lines to be added in the new version.
261
sha -- SHA-1 of the file, if known. This is trusted to be
264
from bzrlib.osutils import sha_strings
266
assert isinstance(name, basestring)
268
sha1 = sha_strings(text)
269
if name in self._name_map:
270
return self._check_repeated_add(name, parents, text, sha1)
272
parents = map(self.maybe_lookup, parents)
196
273
self._check_versions(parents)
197
274
## self._check_lines(text)
198
275
new_version = len(self._parents)
206
# if we abort after here the weave will be corrupt
207
self._parents.append(frozenset(parents))
278
# if we abort after here the (in-memory) weave will be corrupt because only
279
# some fields are updated
280
self._parents.append(parents[:])
208
281
self._sha1s.append(sha1)
282
self._names.append(name)
283
self._name_map[name] = new_version
287
362
# we don't destroy ourselves
289
364
self._weave[i:i] = ([('{', new_version)]
291
+ [('}', new_version)])
292
367
offset += 2 + (j2 - j1)
294
369
return new_version
371
def add_identical(self, old_rev_id, new_rev_id, parents):
372
"""Add an identical text to old_rev_id as new_rev_id."""
373
old_lines = self.get(self.lookup(old_rev_id))
374
self.add(new_rev_id, parents, old_lines)
297
376
def inclusions(self, versions):
298
377
"""Return set of all ancestors of given version(s)."""
299
378
i = set(versions)
304
# include all its parents
305
i.update(self._parents[v])
309
raise ValueError("version %d not present in weave" % v)
379
for v in xrange(max(versions), 0, -1):
381
# include all its parents
382
i.update(self._parents[v])
384
## except IndexError:
385
## raise ValueError("version %d not present in weave" % v)
388
def parents(self, version):
389
return self._parents[version]
392
def parent_names(self, version):
393
"""Return version names for parents of a version."""
394
return map(self.idx_to_name, self._parents[self.lookup(version)])
312
397
def minimal_parents(self, version):
450
537
result.append((istack[-1], lineno, l))
454
raise WFE("unclosed insertion blocks at end of weave",
540
raise WeaveFormatError("unclosed insertion blocks "
541
"at end of weave: %s" % istack)
457
raise WFE("unclosed deletion blocks at end of weave",
543
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
464
def get_iter(self, version):
548
def get_iter(self, name_or_index):
465
549
"""Yield lines for the specified version."""
466
for origin, lineno, line in self._extract([version]):
550
incls = [self.maybe_lookup(name_or_index)]
551
for origin, lineno, line in self._extract(incls):
470
def get(self, index):
471
return list(self.get_iter(index))
555
def get_text(self, name_or_index):
556
return ''.join(self.get_iter(name_or_index))
557
assert isinstance(version, int)
560
def get_lines(self, name_or_index):
561
return list(self.get_iter(name_or_index))
474
567
def mash_iter(self, included):
475
568
"""Return composed version of multiple included versions."""
569
included = map(self.maybe_lookup, included)
476
570
for origin, lineno, text in self._extract(included):
664
743
'killed-both'), \
674
"""Show some text information about the weave."""
675
print '%6s %40s %20s' % ('ver', 'sha1', 'parents')
676
for i in (6, 40, 20):
747
def join(self, other):
749
"""Integrate versions from other into this weave.
751
The resulting weave contains all the history of both weaves;
752
any version you could retrieve from either self or other can be
753
retrieved from self after this call.
755
It is illegal for the two weaves to contain different values
756
or different parents for any version. See also reweave().
758
if other.numversions() == 0:
759
return # nothing to update, easy
760
# two loops so that we do not change ourselves before verifying it
762
# work through in index order to make sure we get all dependencies
763
for other_idx, name in enumerate(other._names):
764
self._check_version_consistent(other, other_idx, name)
769
for other_idx, name in enumerate(other._names):
770
# TODO: If all the parents of the other version are already
771
# present then we can avoid some work by just taking the delta
772
# and adjusting the offsets.
773
new_parents = self._imported_parents(other, other_idx)
774
sha1 = other._sha1s[other_idx]
778
if name in self._names:
779
idx = self.lookup(name)
780
n1 = map(other.idx_to_name, other._parents[other_idx] )
781
n2 = map(self.idx_to_name, self._parents[other_idx] )
782
if sha1 == self._sha1s[idx] and n1 == n2:
786
lines = other.get_lines(other_idx)
787
self.add(name, new_parents, lines, sha1)
789
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
790
merged,processed,self._weave_name, time.time( )-time0))
795
def _imported_parents(self, other, other_idx):
796
"""Return list of parents in self corresponding to indexes in other."""
798
for parent_idx in other._parents[other_idx]:
799
parent_name = other._names[parent_idx]
800
if parent_name not in self._names:
801
# should not be possible
802
raise WeaveError("missing parent {%s} of {%s} in %r"
803
% (parent_name, other._name_map[other_idx], self))
804
new_parents.append(self._name_map[parent_name])
807
def _check_version_consistent(self, other, other_idx, name):
808
"""Check if a version in consistent in this and other.
810
To be consistent it must have:
813
* the same direct parents (by name, not index, and disregarding
816
If present & correct return True;
817
if not present in self return False;
818
if inconsistent raise error."""
819
this_idx = self._name_map.get(name, -1)
821
if self._sha1s[this_idx] != other._sha1s[other_idx]:
822
raise WeaveError("inconsistent texts for version {%s} "
823
"when joining weaves"
825
self_parents = self._parents[this_idx]
826
other_parents = other._parents[other_idx]
827
n1 = [self._names[i] for i in self_parents]
828
n2 = [other._names[i] for i in other_parents]
832
raise WeaveParentMismatch("inconsistent parents "
833
"for version {%s}: %s vs %s" % (name, n1, n2))
839
def reweave(self, other):
840
"""Reweave self with other."""
841
new_weave = reweave(self, other)
842
for attr in self.__slots__:
843
setattr(self, attr, getattr(new_weave, attr))
847
"""Combine two weaves and return the result.
849
This works even if a revision R has different parents in
850
wa and wb. In the resulting weave all the parents are given.
852
This is done by just building up a new weave, maintaining ordering
853
of the versions in the two inputs. More efficient approaches
854
might be possible but it should only be necessary to do
855
this operation rarely, when a new previously ghost version is
860
queue_a = range(wa.numversions())
861
queue_b = range(wb.numversions())
862
# first determine combined parents of all versions
863
# map from version name -> all parent names
864
combined_parents = _reweave_parent_graphs(wa, wb)
865
mutter("combined parents: %r", combined_parents)
866
order = topo_sort(combined_parents.iteritems())
867
mutter("order to reweave: %r", order)
869
if name in wa._name_map:
870
lines = wa.get_lines(name)
871
if name in wb._name_map:
872
assert lines == wb.get_lines(name)
874
lines = wb.get_lines(name)
875
wr.add(name, combined_parents[name], lines)
879
def _reweave_parent_graphs(wa, wb):
880
"""Return combined parent ancestry for two weaves.
882
Returned as a list of (version_name, set(parent_names))"""
884
for weave in [wa, wb]:
885
for idx, name in enumerate(weave._names):
886
p = combined.setdefault(name, set())
887
p.update(map(weave.idx_to_name, weave._parents[idx]))
892
"""Show the weave's table-of-contents"""
893
print '%6s %50s %10s %10s' % ('ver', 'name', 'sha1', 'parents')
894
for i in (6, 50, 10, 10):
679
897
for i in range(w.numversions()):
680
898
sha1 = w._sha1s[i]
681
print '%6d %40s %s' % (i, sha1, ' '.join(map(str, w._parents[i])))
685
def weave_stats(weave_file):
686
from bzrlib.progress import ProgressBar
900
parent_str = ' '.join(map(str, w._parents[i]))
901
print '%6d %-50.50s %10.10s %s' % (i, name, sha1, parent_str)
905
def weave_stats(weave_file, pb):
687
906
from bzrlib.weavefile import read_weave
691
908
wf = file(weave_file, 'rb')
692
909
w = read_weave(wf)
693
910
# FIXME: doesn't work on pipes
added
removed
bzrlib/weave.py
