64
59
'bzrlib.knit', 'FTAnnotatedToUnannotated')
65
60
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
61
'bzrlib.knit', 'FTAnnotatedToFullText')
67
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
71
64
class ContentFactory(object):
72
65
"""Abstract interface for insertion and retrieval from a VersionedFile.
74
67
:ivar sha1: None, or the sha1 of the content fulltext.
75
68
:ivar storage_kind: The native storage kind of this factory. One of
76
69
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
91
84
self.parents = None
94
class ChunkedContentFactory(ContentFactory):
95
"""Static data content factory.
97
This takes a 'chunked' list of strings. The only requirement on 'chunked' is
98
that ''.join(lines) becomes a valid fulltext. A tuple of a single string
99
satisfies this, as does a list of lines.
101
:ivar sha1: None, or the sha1 of the content fulltext.
102
:ivar storage_kind: The native storage kind of this factory. Always
104
:ivar key: The key of this content. Each key is a tuple with a single
106
:ivar parents: A tuple of parent keys for self.key. If the object has
107
no parent information, None (as opposed to () for an empty list of
111
def __init__(self, key, parents, sha1, chunks):
112
"""Create a ContentFactory."""
114
self.storage_kind = 'chunked'
116
self.parents = parents
117
self._chunks = chunks
119
def get_bytes_as(self, storage_kind):
120
if storage_kind == 'chunked':
122
elif storage_kind == 'fulltext':
123
return ''.join(self._chunks)
124
raise errors.UnavailableRepresentation(self.key, storage_kind,
128
87
class FulltextContentFactory(ContentFactory):
129
88
"""Static data content factory.
131
90
This takes a fulltext when created and just returns that during
132
91
get_bytes_as('fulltext').
134
93
:ivar sha1: None, or the sha1 of the content fulltext.
135
94
:ivar storage_kind: The native storage kind of this factory. Always
468
419
if isinstance(version_ids, basestring):
469
420
version_ids = [version_ids]
470
421
raise NotImplementedError(self.get_ancestry)
472
423
def get_ancestry_with_ghosts(self, version_ids):
473
424
"""Return a list of all ancestors of given version(s). This
474
425
will not include the null revision.
476
427
Must raise RevisionNotPresent if any of the given versions are
477
428
not present in file history.
479
430
Ghosts that are known about will be included in ancestry list,
480
431
but are not explicitly marked.
482
433
raise NotImplementedError(self.get_ancestry_with_ghosts)
484
435
def get_parent_map(self, version_ids):
485
436
"""Get a map of the parents of version_ids.
611
562
return self._backing_vf.keys()
614
class OrderingVersionedFilesDecorator(RecordingVersionedFilesDecorator):
615
"""A VF that records calls, and returns keys in specific order.
617
:ivar calls: A list of the calls made; can be reset at any time by
621
def __init__(self, backing_vf, key_priority):
622
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
624
:param backing_vf: The versioned file to answer all methods.
625
:param key_priority: A dictionary defining what order keys should be
626
returned from an 'unordered' get_record_stream request.
627
Keys with lower priority are returned first, keys not present in
628
the map get an implicit priority of 0, and are returned in
629
lexicographical order.
631
RecordingVersionedFilesDecorator.__init__(self, backing_vf)
632
self._key_priority = key_priority
634
def get_record_stream(self, keys, sort_order, include_delta_closure):
635
self.calls.append(("get_record_stream", list(keys), sort_order,
636
include_delta_closure))
637
if sort_order == 'unordered':
639
return (self._key_priority.get(key, 0), key)
640
# Use a defined order by asking for the keys one-by-one from the
642
for key in sorted(keys, key=sort_key):
643
for record in self._backing_vf.get_record_stream([key],
644
'unordered', include_delta_closure):
647
for record in self._backing_vf.get_record_stream(keys, sort_order,
648
include_delta_closure):
652
565
class KeyMapper(object):
653
566
"""KeyMappers map between keys and underlying partitioned storage."""
837
749
raise NotImplementedError(self.add_lines)
839
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
840
"""Add a text to the store.
842
This is a private function for use by CommitBuilder.
844
:param key: The key tuple of the text to add. If the last element is
845
None, a CHK string will be generated during the addition.
846
:param parents: The parents key tuples of the text to add.
847
:param text: A string containing the text to be committed.
848
:param nostore_sha: Raise ExistingContent and do not add the lines to
849
the versioned file if the digest of the lines matches this.
850
:param random_id: If True a random id has been selected rather than
851
an id determined by some deterministic process such as a converter
852
from a foreign VCS. When True the backend may choose not to check
853
for uniqueness of the resulting key within the versioned file, so
854
this should only be done when the result is expected to be unique
856
:param check_content: If True, the lines supplied are verified to be
857
bytestrings that are correctly formed lines.
858
:return: The text sha1, the number of bytes in the text, and an opaque
859
representation of the inserted version which can be provided
860
back to future _add_text calls in the parent_texts dictionary.
862
# The default implementation just thunks over to .add_lines(),
863
# inefficient, but it works.
864
return self.add_lines(key, parents, osutils.split_lines(text),
865
nostore_sha=nostore_sha,
869
751
def add_mpdiffs(self, records):
870
752
"""Add mpdiffs to this VersionedFile.
884
766
if not mpvf.has_version(p))
885
767
# It seems likely that adding all the present parents as fulltexts can
886
768
# easily exhaust memory.
887
chunks_to_lines = osutils.chunks_to_lines
769
split_lines = osutils.split_lines
888
770
for record in self.get_record_stream(needed_parents, 'unordered',
890
772
if record.storage_kind == 'absent':
892
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
774
mpvf.add_version(split_lines(record.get_bytes_as('fulltext')),
894
776
for (key, parent_keys, expected_sha1, mpdiff), lines in\
895
777
zip(records, mpvf.get_line_list(versions)):
913
795
raise NotImplementedError(self.annotate)
915
797
def check(self, progress_bar=None):
916
"""Check this object for integrity.
918
:param progress_bar: A progress bar to output as the check progresses.
919
:param keys: Specific keys within the VersionedFiles to check. When
920
this parameter is not None, check() becomes a generator as per
921
get_record_stream. The difference to get_record_stream is that
922
more or deeper checks will be performed.
923
:return: None, or if keys was supplied a generator as per
798
"""Check this object for integrity."""
926
799
raise NotImplementedError(self.check)
929
802
def check_not_reserved_id(version_id):
930
803
revision.check_not_reserved_id(version_id)
932
def clear_cache(self):
933
"""Clear whatever caches this VersionedFile holds.
935
This is generally called after an operation has been performed, when we
936
don't expect to be using this versioned file again soon.
939
805
def _check_lines_not_unicode(self, lines):
940
806
"""Check that lines being added to a versioned file are not unicode."""
941
807
for line in lines:
998
850
has_key = index._has_key_from_parent_map
1000
def get_missing_compression_parent_keys(self):
1001
"""Return an iterable of keys of missing compression parents.
1003
Check this after calling insert_record_stream to find out if there are
1004
any missing compression parents. If there are, the records that
1005
depend on them are not able to be inserted safely. The precise
1006
behaviour depends on the concrete VersionedFiles class in use.
1008
Classes that do not support this will raise NotImplementedError.
1010
raise NotImplementedError(self.get_missing_compression_parent_keys)
1012
852
def insert_record_stream(self, stream):
1013
853
"""Insert a record stream into this container.
1015
:param stream: A stream of records to insert.
855
:param stream: A stream of records to insert.
1017
857
:seealso VersionedFile.get_record_stream:
1159
999
result.append((prefix + (origin,), line))
1162
def get_annotator(self):
1163
return annotate.Annotator(self)
1165
def check(self, progress_bar=None, keys=None):
1002
def check(self, progress_bar=None):
1166
1003
"""See VersionedFiles.check()."""
1167
# XXX: This is over-enthusiastic but as we only thunk for Weaves today
1168
# this is tolerable. Ideally we'd pass keys down to check() and
1169
# have the older VersiondFile interface updated too.
1170
1004
for prefix, vf in self._iter_all_components():
1172
if keys is not None:
1173
return self.get_record_stream(keys, 'unordered', True)
1175
1007
def get_parent_map(self, keys):
1176
1008
"""Get a map of the parents of keys.
1478
1311
elif state == 'conflicted-b':
1479
1312
ch_b = ch_a = True
1480
1313
lines_b.append(line)
1481
elif state == 'killed-both':
1482
# This counts as a change, even though there is no associated
1486
1315
if state not in ('irrelevant', 'ghost-a', 'ghost-b',
1316
'killed-base', 'killed-both'):
1488
1317
raise AssertionError(state)
1489
1318
for struct in outstanding_struct():
1492
def base_from_plan(self):
1493
"""Construct a BASE file from the plan text."""
1495
for state, line in self.plan:
1496
if state in ('killed-a', 'killed-b', 'killed-both', 'unchanged'):
1497
# If unchanged, then this line is straight from base. If a or b
1498
# or both killed the line, then it *used* to be in base.
1499
base_lines.append(line)
1501
if state not in ('killed-base', 'irrelevant',
1502
'ghost-a', 'ghost-b',
1504
'conflicted-a', 'conflicted-b'):
1505
# killed-base, irrelevant means it doesn't apply
1506
# ghost-a/ghost-b are harder to say for sure, but they
1507
# aren't in the 'inc_c' which means they aren't in the
1508
# shared base of a & b. So we don't include them. And
1509
# obviously if the line is newly inserted, it isn't in base
1511
# If 'conflicted-a' or b, then it is new vs one base, but
1512
# old versus another base. However, if we make it present
1513
# in the base, it will be deleted from the target, and it
1514
# seems better to get a line doubled in the merge result,
1515
# rather than have it deleted entirely.
1516
# Example, each node is the 'text' at that point:
1524
# There was a criss-cross conflict merge. Both sides
1525
# include the other, but put themselves first.
1526
# Weave marks this as a 'clean' merge, picking OTHER over
1527
# THIS. (Though the details depend on order inserted into
1529
# LCA generates a plan:
1530
# [('unchanged', M),
1531
# ('conflicted-b', b),
1533
# ('conflicted-a', b),
1535
# If you mark 'conflicted-*' as part of BASE, then a 3-way
1536
# merge tool will cleanly generate "MaN" (as BASE vs THIS
1537
# removes one 'b', and BASE vs OTHER removes the other)
1538
# If you include neither, 3-way creates a clean "MbabN" as
1539
# THIS adds one 'b', and OTHER does too.
1540
# It seems that having the line 2 times is better than
1541
# having it omitted. (Easier to manually delete than notice
1542
# it needs to be added.)
1543
raise AssertionError('Unknown state: %s' % (state,))
1547
1322
class WeaveMerge(PlanWeaveMerge):
1548
1323
"""Weave merge that takes a VersionedFile and two versions as its input."""
1550
def __init__(self, versionedfile, ver_a, ver_b,
1325
def __init__(self, versionedfile, ver_a, ver_b,
1551
1326
a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1552
1327
plan = versionedfile.plan_merge(ver_a, ver_b)
1553
1328
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1556
1331
class VirtualVersionedFiles(VersionedFiles):
1557
"""Dummy implementation for VersionedFiles that uses other functions for
1332
"""Dummy implementation for VersionedFiles that uses other functions for
1558
1333
obtaining fulltexts and parent maps.
1560
This is always on the bottom of the stack and uses string keys
1335
This is always on the bottom of the stack and uses string keys
1561
1336
(rather than tuples) internally.
1609
1384
if lines is not None:
1610
1385
if not isinstance(lines, list):
1611
1386
raise AssertionError
1612
yield ChunkedContentFactory((k,), None,
1387
yield FulltextContentFactory((k,), None,
1613
1388
sha1=osutils.sha_strings(lines),
1389
text=''.join(lines))
1616
1391
yield AbsentContentFactory((k,))
1618
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1619
"""See VersionedFile.iter_lines_added_or_present_in_versions()."""
1620
for i, (key,) in enumerate(keys):
1622
pb.update("Finding changed lines", i, len(keys))
1623
for l in self._get_lines(key):
1627
def network_bytes_to_kind_and_offset(network_bytes):
1628
"""Strip of a record kind from the front of network_bytes.
1630
:param network_bytes: The bytes of a record.
1631
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1633
line_end = network_bytes.find('\n')
1634
storage_kind = network_bytes[:line_end]
1635
return storage_kind, line_end + 1
1638
class NetworkRecordStream(object):
1639
"""A record_stream which reconstitures a serialised stream."""
1641
def __init__(self, bytes_iterator):
1642
"""Create a NetworkRecordStream.
1644
:param bytes_iterator: An iterator of bytes. Each item in this
1645
iterator should have been obtained from a record_streams'
1646
record.get_bytes_as(record.storage_kind) call.
1648
self._bytes_iterator = bytes_iterator
1649
self._kind_factory = {
1650
'fulltext': fulltext_network_to_record,
1651
'groupcompress-block': groupcompress.network_block_to_records,
1652
'knit-ft-gz': knit.knit_network_to_record,
1653
'knit-delta-gz': knit.knit_network_to_record,
1654
'knit-annotated-ft-gz': knit.knit_network_to_record,
1655
'knit-annotated-delta-gz': knit.knit_network_to_record,
1656
'knit-delta-closure': knit.knit_delta_closure_to_records,
1662
:return: An iterator as per VersionedFiles.get_record_stream().
1664
for bytes in self._bytes_iterator:
1665
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1666
for record in self._kind_factory[storage_kind](
1667
storage_kind, bytes, line_end):
1671
def fulltext_network_to_record(kind, bytes, line_end):
1672
"""Convert a network fulltext record to record."""
1673
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1674
record_meta = bytes[line_end+4:line_end+4+meta_len]
1675
key, parents = bencode.bdecode_as_tuple(record_meta)
1676
if parents == 'nil':
1678
fulltext = bytes[line_end+4+meta_len:]
1679
return [FulltextContentFactory(key, parents, None, fulltext)]
1682
def _length_prefix(bytes):
1683
return struct.pack('!L', len(bytes))
1686
def record_to_fulltext_bytes(record):
1687
if record.parents is None:
1690
parents = record.parents
1691
record_meta = bencode.bencode((record.key, parents))
1692
record_content = record.get_bytes_as('fulltext')
1693
return "fulltext\n%s%s%s" % (
1694
_length_prefix(record_meta), record_meta, record_content)
1697
def sort_groupcompress(parent_map):
1698
"""Sort and group the keys in parent_map into groupcompress order.
1700
groupcompress is defined (currently) as reverse-topological order, grouped
1703
:return: A sorted-list of keys
1705
# gc-optimal ordering is approximately reverse topological,
1706
# properly grouped by file-id.
1708
for item in parent_map.iteritems():
1710
if isinstance(key, str) or len(key) == 1:
1715
per_prefix_map[prefix].append(item)
1717
per_prefix_map[prefix] = [item]
1720
for prefix in sorted(per_prefix_map):
1721
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
added
removed
bzrlib/versionedfile.py
