74
44
# TODO: Parallel-extract that passes back each line along with a
75
45
# description of which revisions include it. Nice for checking all
81
class WeaveError(Exception):
82
"""Exception in processing weave"""
85
class WeaveFormatError(WeaveError):
86
"""Weave invariant violated"""
46
# shas or calculating stats in parallel.
48
# TODO: Using a single _extract routine and then processing the output
49
# is probably inefficient. It's simple enough that we can afford to
50
# have slight specializations for different ways its used: annotate,
51
# basis for add, get, etc.
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of
65
# something in the list.) We should eliminate them. This can
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
72
from difflib import SequenceMatcher
74
from bzrlib.trace import mutter
75
from bzrlib.errors import (WeaveError, WeaveFormatError, WeaveParentMismatch,
76
WeaveRevisionNotPresent, WeaveRevisionAlreadyPresent)
77
import bzrlib.errors as errors
78
from bzrlib.tsort import topo_sort
89
81
class Weave(object):
90
82
"""weave - versioned text file storage.
160
154
each version; the parent's parents are implied.
163
List of hex SHA-1 of each version, or None if not recorded.
157
List of hex SHA-1 of each version.
160
List of symbolic names for each version. Each should be unique.
163
For each name, the version number.
166
Descriptive name of this weave; typically the filename if known.
166
__slots__ = ['_weave', '_parents', '_sha1s']
170
__slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map',
173
def __init__(self, weave_name=None):
170
175
self._parents = []
179
self._weave_name = weave_name
182
return "Weave(%r)" % self._weave_name
186
"""Return a deep copy of self.
188
The copy can be modified without affecting the original weave."""
190
other._weave = self._weave[:]
191
other._parents = self._parents[:]
192
other._sha1s = self._sha1s[:]
193
other._names = self._names[:]
194
other._name_map = self._name_map.copy()
195
other._weave_name = self._weave_name
174
198
def __eq__(self, other):
175
199
if not isinstance(other, Weave):
177
201
return self._parents == other._parents \
178
and self._weave == other._weave
202
and self._weave == other._weave \
203
and self._sha1s == other._sha1s
181
206
def __ne__(self, other):
182
207
return not self.__eq__(other)
185
def add(self, parents, text):
209
def __contains__(self, name):
210
return self._name_map.has_key(name)
212
def maybe_lookup(self, name_or_index):
213
"""Convert possible symbolic name to index, or pass through indexes."""
214
if isinstance(name_or_index, (int, long)):
217
return self.lookup(name_or_index)
220
def lookup(self, name):
221
"""Convert symbolic version name to index."""
223
return self._name_map[name]
225
raise WeaveRevisionNotPresent(name, self)
228
return self._names[:]
230
def iter_names(self):
231
"""Yield a list of all names in this weave."""
232
return iter(self._names)
234
def idx_to_name(self, version):
235
return self._names[version]
237
def _check_repeated_add(self, name, parents, text, sha1):
238
"""Check that a duplicated add is OK.
240
If it is, return the (old) index; otherwise raise an exception.
242
idx = self.lookup(name)
243
if sorted(self._parents[idx]) != sorted(parents) \
244
or sha1 != self._sha1s[idx]:
245
raise WeaveRevisionAlreadyPresent(name, self)
248
def add(self, name, parents, text, sha1=None):
186
249
"""Add a single text on top of the weave.
188
251
Returns the index number of the newly added version.
254
Symbolic name for this version.
255
(Typically the revision-id of the revision that added it.)
191
258
List or set of direct parent version numbers.
194
Sequence of lines to be added in the new version."""
261
Sequence of lines to be added in the new version.
263
sha -- SHA-1 of the file, if known. This is trusted to be
266
from bzrlib.osutils import sha_strings
268
assert isinstance(name, basestring)
270
sha1 = sha_strings(text)
271
if name in self._name_map:
272
return self._check_repeated_add(name, parents, text, sha1)
274
parents = map(self.maybe_lookup, parents)
196
275
self._check_versions(parents)
197
276
## self._check_lines(text)
198
277
new_version = len(self._parents)
206
# if we abort after here the weave will be corrupt
207
self._parents.append(frozenset(parents))
280
# if we abort after here the (in-memory) weave will be corrupt because only
281
# some fields are updated
282
# XXX: FIXME implement a succeed-or-fail of the rest of this routine.
283
# - Robert Collins 20060226
284
self._parents.append(parents[:])
208
285
self._sha1s.append(sha1)
286
self._names.append(name)
287
self._name_map[name] = new_version
450
546
result.append((istack[-1], lineno, l))
454
raise WFE("unclosed insertion blocks at end of weave",
549
raise WeaveFormatError("unclosed insertion blocks "
550
"at end of weave: %s" % istack)
457
raise WFE("unclosed deletion blocks at end of weave",
552
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
464
def get_iter(self, version):
557
def get_iter(self, name_or_index):
465
558
"""Yield lines for the specified version."""
466
for origin, lineno, line in self._extract([version]):
559
incls = [self.maybe_lookup(name_or_index)]
564
# We don't have sha1 sums for multiple entries
566
for origin, lineno, line in self._extract(incls):
470
def get(self, index):
471
return list(self.get_iter(index))
571
expected_sha1 = self._sha1s[index]
572
measured_sha1 = cur_sha.hexdigest()
573
if measured_sha1 != expected_sha1:
574
raise errors.WeaveInvalidChecksum(
575
'file %s, revision %s, expected: %s, measured %s'
576
% (self._weave_name, self._names[index],
577
expected_sha1, measured_sha1))
580
def get_text(self, name_or_index):
581
return ''.join(self.get_iter(name_or_index))
582
assert isinstance(version, int)
585
def get_lines(self, name_or_index):
586
return list(self.get_iter(name_or_index))
592
def get_sha1(self, name):
593
"""Get the stored sha1 sum for the given revision.
595
:param name: The name of the version to lookup
597
return self._sha1s[self.lookup(name)]
474
599
def mash_iter(self, included):
475
600
"""Return composed version of multiple included versions."""
601
included = map(self.maybe_lookup, included)
476
602
for origin, lineno, text in self._extract(included):
506
631
raise WeaveFormatError("invalid included version %d for index %d"
507
632
% (inclusions[-1], version))
509
# try extracting all versions; this is a bit slow and parallel
510
# extraction could be used
634
# try extracting all versions; parallel extraction is used
512
635
nv = self.numversions()
636
sha1s = [sha.new() for i in range(nv)]
637
texts = [[] for i in range(nv)]
640
# For creating the ancestry, IntSet is much faster (3.7s vs 0.17s)
641
# The problem is that set membership is much more expensive
643
for p in self._parents[i]:
644
new_inc.update(inclusions[p])
646
#assert set(new_inc) == self.inclusions([i]), 'failed %s != %s' % (new_inc, self.inclusions([i]))
647
inclusions.append(new_inc)
649
nlines = len(self._weave)
651
update_text = 'checking weave'
653
short_name = os.path.basename(self._weave_name)
654
update_text = 'checking %s' % (short_name,)
655
update_text = update_text[:25]
657
for lineno, insert, deleteset, line in self._walk():
659
progress_bar.update(update_text, lineno, nlines)
661
for j, j_inc in enumerate(inclusions):
662
# The active inclusion must be an ancestor,
663
# and no ancestors must have deleted this line,
664
# because we don't support resurrection.
665
if (insert in j_inc) and not (deleteset & j_inc):
666
sha1s[j].update(line)
513
668
for version in range(nv):
515
progress_bar.update('checking text', version, nv)
517
for l in self.get_iter(version):
669
hd = sha1s[version].hexdigest()
520
670
expected = self._sha1s[version]
521
671
if hd != expected:
522
raise WeaveError("mismatched sha1 for version %d; "
523
"got %s, expected %s"
524
% (version, hd, expected))
672
raise errors.WeaveInvalidChecksum(
673
"mismatched sha1 for version %s: "
674
"got %s, expected %s"
675
% (self._names[version], hd, expected))
526
677
# TODO: check insertions are properly nested, that there are
527
678
# no lines outside of insertion blocks, that deletions are
528
679
# properly paired, etc.
532
def merge(self, merge_versions):
533
"""Automerge and mark conflicts between versions.
535
This returns a sequence, each entry describing alternatives
536
for a chunk of the file. Each of the alternatives is given as
539
If there is a chunk of the file where there's no diagreement,
540
only one alternative is given.
543
# approach: find the included versions common to all the
545
raise NotImplementedError()
549
681
def _delta(self, included, lines):
550
682
"""Return changes from basis to new revision.
664
800
'killed-both'), \
674
"""Show some text information about the weave."""
675
print '%6s %40s %20s' % ('ver', 'sha1', 'parents')
676
for i in (6, 40, 20):
804
def join(self, other, pb=None, msg=None):
806
"""Integrate versions from other into this weave.
808
The resulting weave contains all the history of both weaves;
809
any version you could retrieve from either self or other can be
810
retrieved from self after this call.
812
It is illegal for the two weaves to contain different values
813
or different parents for any version. See also reweave().
815
:param other: The other weave to pull into this one
816
:param pb: An optional progress bar
817
:param msg: An optional message to display for progress
819
if other.numversions() == 0:
820
return # nothing to update, easy
821
# two loops so that we do not change ourselves before verifying it
823
# work through in index order to make sure we get all dependencies
826
for other_idx, name in enumerate(other._names):
827
self._check_version_consistent(other, other_idx, name)
828
sha1 = other._sha1s[other_idx]
832
if name in self._name_map:
833
idx = self.lookup(name)
834
n1 = set(map(other.idx_to_name, other._parents[other_idx]))
835
n2 = set(map(self.idx_to_name, self._parents[idx]))
836
if sha1 == self._sha1s[idx] and n1 == n2:
839
names_to_join.append((other_idx, name))
846
for other_idx, name in names_to_join:
847
# TODO: If all the parents of the other version are already
848
# present then we can avoid some work by just taking the delta
849
# and adjusting the offsets.
850
new_parents = self._imported_parents(other, other_idx)
851
sha1 = other._sha1s[other_idx]
856
pb.update(msg, merged, len(names_to_join))
858
lines = other.get_lines(other_idx)
859
self.add(name, new_parents, lines, sha1)
861
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
862
merged, processed, self._weave_name, time.time( )-time0))
864
def _imported_parents(self, other, other_idx):
865
"""Return list of parents in self corresponding to indexes in other."""
867
for parent_idx in other._parents[other_idx]:
868
parent_name = other._names[parent_idx]
869
if parent_name not in self._names:
870
# should not be possible
871
raise WeaveError("missing parent {%s} of {%s} in %r"
872
% (parent_name, other._name_map[other_idx], self))
873
new_parents.append(self._name_map[parent_name])
876
def _check_version_consistent(self, other, other_idx, name):
877
"""Check if a version in consistent in this and other.
879
To be consistent it must have:
882
* the same direct parents (by name, not index, and disregarding
885
If present & correct return True;
886
if not present in self return False;
887
if inconsistent raise error."""
888
this_idx = self._name_map.get(name, -1)
890
if self._sha1s[this_idx] != other._sha1s[other_idx]:
891
raise WeaveError("inconsistent texts for version {%s} "
892
"when joining weaves"
894
self_parents = self._parents[this_idx]
895
other_parents = other._parents[other_idx]
896
n1 = set([self._names[i] for i in self_parents])
897
n2 = set([other._names[i] for i in other_parents])
899
raise WeaveParentMismatch("inconsistent parents "
900
"for version {%s}: %s vs %s" % (name, n1, n2))
906
def reweave(self, other, pb=None, msg=None):
907
"""Reweave self with other.
909
:param other: The other weave to merge
910
:param pb: An optional progress bar, indicating how far done we are
911
:param msg: An optional message for the progress
913
new_weave = reweave(self, other, pb=pb, msg=msg)
914
for attr in self.__slots__:
915
setattr(self, attr, getattr(new_weave, attr))
918
def reweave(wa, wb, pb=None, msg=None):
919
"""Combine two weaves and return the result.
921
This works even if a revision R has different parents in
922
wa and wb. In the resulting weave all the parents are given.
924
This is done by just building up a new weave, maintaining ordering
925
of the versions in the two inputs. More efficient approaches
926
might be possible but it should only be necessary to do
927
this operation rarely, when a new previously ghost version is
930
:param pb: An optional progress bar, indicating how far done we are
931
:param msg: An optional message for the progress
935
queue_a = range(wa.numversions())
936
queue_b = range(wb.numversions())
937
# first determine combined parents of all versions
938
# map from version name -> all parent names
939
combined_parents = _reweave_parent_graphs(wa, wb)
940
mutter("combined parents: %r", combined_parents)
941
order = topo_sort(combined_parents.iteritems())
942
mutter("order to reweave: %r", order)
947
for idx, name in enumerate(order):
949
pb.update(msg, idx, len(order))
950
if name in wa._name_map:
951
lines = wa.get_lines(name)
952
if name in wb._name_map:
953
lines_b = wb.get_lines(name)
955
mutter('Weaves differ on content. rev_id {%s}', name)
956
mutter('weaves: %s, %s', wa._weave_name, wb._weave_name)
958
lines = list(difflib.unified_diff(lines, lines_b,
959
wa._weave_name, wb._weave_name))
960
mutter('lines:\n%s', ''.join(lines))
961
raise errors.WeaveTextDiffers(name, wa, wb)
963
lines = wb.get_lines(name)
964
wr.add(name, combined_parents[name], lines)
968
def _reweave_parent_graphs(wa, wb):
969
"""Return combined parent ancestry for two weaves.
971
Returned as a list of (version_name, set(parent_names))"""
973
for weave in [wa, wb]:
974
for idx, name in enumerate(weave._names):
975
p = combined.setdefault(name, set())
976
p.update(map(weave.idx_to_name, weave._parents[idx]))
981
"""Show the weave's table-of-contents"""
982
print '%6s %50s %10s %10s' % ('ver', 'name', 'sha1', 'parents')
983
for i in (6, 50, 10, 10):
679
986
for i in range(w.numversions()):
680
987
sha1 = w._sha1s[i]
681
print '%6d %40s %s' % (i, sha1, ' '.join(map(str, w._parents[i])))
685
def weave_stats(weave_file):
686
from bzrlib.progress import ProgressBar
989
parent_str = ' '.join(map(str, w._parents[i]))
990
print '%6d %-50.50s %10.10s %s' % (i, name, sha1, parent_str)
994
def weave_stats(weave_file, pb):
687
995
from bzrlib.weavefile import read_weave
691
997
wf = file(weave_file, 'rb')
692
998
w = read_weave(wf)
693
999
# FIXME: doesn't work on pipes
added
removed
bzrlib/weave.py
