17
20
"""Versioned text file storage api."""
19
from __future__ import absolute_import
21
22
from copy import copy
23
from cStringIO import StringIO
25
26
from zlib import adler32
27
from ..lazy_import import lazy_import
28
from bzrlib.lazy_import import lazy_import
28
29
lazy_import(globals(), """
32
35
graph as _mod_graph,
39
from breezy.bzr import (
45
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
46
from bzrlib.transport.memory import MemoryTransport
48
from ..registry import Registry
49
from ..sixish import (
55
from ..textmerge import TextMerge
48
from bzrlib.registry import Registry
49
from bzrlib.symbol_versioning import *
50
from bzrlib.textmerge import TextMerge
51
from bzrlib import bencode
58
54
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
56
'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
59
59
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
'breezy.bzr.knit', 'DeltaAnnotatedToUnannotated')
60
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
'bzrlib.knit', 'DeltaAnnotatedToFullText')
61
63
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
62
'breezy.bzr.knit', 'FTAnnotatedToUnannotated')
63
for target_storage_kind in ('fulltext', 'chunked', 'lines'):
64
adapter_registry.register_lazy(('knit-delta-gz', target_storage_kind), 'breezy.bzr.knit',
65
'DeltaPlainToFullText')
66
adapter_registry.register_lazy(('knit-ft-gz', target_storage_kind), 'breezy.bzr.knit',
68
adapter_registry.register_lazy(('knit-annotated-ft-gz', target_storage_kind),
69
'breezy.bzr.knit', 'FTAnnotatedToFullText')
70
adapter_registry.register_lazy(('knit-annotated-delta-gz', target_storage_kind),
71
'breezy.bzr.knit', 'DeltaAnnotatedToFullText')
74
class UnavailableRepresentation(errors.InternalBzrError):
76
_fmt = ("The encoding '%(wanted)s' is not available for key %(key)s which "
77
"is encoded as '%(native)s'.")
79
def __init__(self, key, wanted, native):
80
errors.InternalBzrError.__init__(self)
86
class ExistingContent(errors.BzrError):
88
_fmt = "The content being inserted is already present."
64
'bzrlib.knit', 'FTAnnotatedToUnannotated')
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
'bzrlib.knit', 'FTAnnotatedToFullText')
67
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
91
71
class ContentFactory(object):
92
72
"""Abstract interface for insertion and retrieval from a VersionedFile.
94
74
:ivar sha1: None, or the sha1 of the content fulltext.
95
:ivar size: None, or the size of the content fulltext.
96
75
:ivar storage_kind: The native storage kind of this factory. One of
97
76
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
98
77
'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
129
106
:ivar parents: A tuple of parent keys for self.key. If the object has
130
107
no parent information, None (as opposed to () for an empty list of
132
:ivar chunks_are_lines: Whether chunks are lines.
135
def __init__(self, key, parents, sha1, chunks, chunks_are_lines=None):
111
def __init__(self, key, parents, sha1, chunks):
136
112
"""Create a ContentFactory."""
138
self.size = sum(map(len, chunks))
139
114
self.storage_kind = 'chunked'
141
116
self.parents = parents
142
117
self._chunks = chunks
143
self._chunks_are_lines = chunks_are_lines
145
119
def get_bytes_as(self, storage_kind):
146
120
if storage_kind == 'chunked':
147
121
return self._chunks
148
122
elif storage_kind == 'fulltext':
149
return b''.join(self._chunks)
150
elif storage_kind == 'lines':
151
if self._chunks_are_lines:
153
return list(osutils.chunks_to_lines(self._chunks))
154
raise UnavailableRepresentation(self.key, storage_kind,
123
return ''.join(self._chunks)
124
raise errors.UnavailableRepresentation(self.key, storage_kind,
157
def iter_bytes_as(self, storage_kind):
158
if storage_kind == 'chunked':
159
return iter(self._chunks)
160
elif storage_kind == 'lines':
161
if self._chunks_are_lines:
162
return iter(self._chunks)
163
return iter(osutils.chunks_to_lines(self._chunks))
164
raise UnavailableRepresentation(self.key, storage_kind,
167
128
class FulltextContentFactory(ContentFactory):
168
129
"""Static data content factory.
196
154
return self._text
197
155
elif storage_kind == 'chunked':
198
156
return [self._text]
199
elif storage_kind == 'lines':
200
return osutils.split_lines(self._text)
201
raise UnavailableRepresentation(self.key, storage_kind,
204
def iter_bytes_as(self, storage_kind):
205
if storage_kind == 'chunked':
206
return iter([self._text])
207
elif storage_kind == 'lines':
208
return iter(osutils.split_lines(self._text))
209
raise UnavailableRepresentation(self.key, storage_kind,
213
class FileContentFactory(ContentFactory):
214
"""File-based content factory.
217
def __init__(self, key, parents, fileobj, sha1=None, size=None):
219
self.parents = parents
221
self.storage_kind = 'file'
225
def get_bytes_as(self, storage_kind):
227
if storage_kind == 'fulltext':
228
return self.file.read()
229
elif storage_kind == 'chunked':
230
return list(osutils.file_iterator(self.file))
231
elif storage_kind == 'lines':
232
return list(self.file.readlines())
233
raise UnavailableRepresentation(self.key, storage_kind,
236
def iter_bytes_as(self, storage_kind):
238
if storage_kind == 'chunked':
239
return osutils.file_iterator(self.file)
240
elif storage_kind == 'lines':
242
raise UnavailableRepresentation(self.key, storage_kind,
157
raise errors.UnavailableRepresentation(self.key, storage_kind,
246
161
class AbsentContentFactory(ContentFactory):
301
class _MPDiffGenerator(object):
302
"""Pull out the functionality for generating mp_diffs."""
304
def __init__(self, vf, keys):
306
# This is the order the keys were requested in
307
self.ordered_keys = tuple(keys)
308
# keys + their parents, what we need to compute the diffs
309
self.needed_keys = ()
310
# Map from key: mp_diff
312
# Map from key: parents_needed (may have ghosts)
314
# Parents that aren't present
315
self.ghost_parents = ()
316
# Map from parent_key => number of children for this text
318
# Content chunks that are cached while we still need them
321
def _find_needed_keys(self):
322
"""Find the set of keys we need to request.
324
This includes all the original keys passed in, and the non-ghost
325
parents of those keys.
327
:return: (needed_keys, refcounts)
328
needed_keys is the set of all texts we need to extract
329
refcounts is a dict of {key: num_children} letting us know when we
330
no longer need to cache a given parent text
332
# All the keys and their parents
333
needed_keys = set(self.ordered_keys)
334
parent_map = self.vf.get_parent_map(needed_keys)
335
self.parent_map = parent_map
336
# TODO: Should we be using a different construct here? I think this
337
# uses difference_update internally, and we expect the result to
339
missing_keys = needed_keys.difference(parent_map)
341
raise errors.RevisionNotPresent(list(missing_keys)[0], self.vf)
342
# Parents that might be missing. They are allowed to be ghosts, but we
343
# should check for them
345
setdefault = refcounts.setdefault
347
for child_key, parent_keys in viewitems(parent_map):
349
# parent_keys may be None if a given VersionedFile claims to
350
# not support graph operations.
352
just_parents.update(parent_keys)
353
needed_keys.update(parent_keys)
354
for p in parent_keys:
355
refcounts[p] = setdefault(p, 0) + 1
356
just_parents.difference_update(parent_map)
357
# Remove any parents that are actually ghosts from the needed set
358
self.present_parents = set(self.vf.get_parent_map(just_parents))
359
self.ghost_parents = just_parents.difference(self.present_parents)
360
needed_keys.difference_update(self.ghost_parents)
361
self.needed_keys = needed_keys
362
self.refcounts = refcounts
363
return needed_keys, refcounts
365
def _compute_diff(self, key, parent_lines, lines):
366
"""Compute a single mp_diff, and store it in self._diffs"""
367
if len(parent_lines) > 0:
368
# XXX: _extract_blocks is not usefully defined anywhere...
369
# It was meant to extract the left-parent diff without
370
# having to recompute it for Knit content (pack-0.92,
371
# etc). That seems to have regressed somewhere
372
left_parent_blocks = self.vf._extract_blocks(key,
373
parent_lines[0], lines)
375
left_parent_blocks = None
376
diff = multiparent.MultiParent.from_lines(lines,
377
parent_lines, left_parent_blocks)
378
self.diffs[key] = diff
380
def _process_one_record(self, key, this_chunks):
382
if key in self.parent_map:
383
# This record should be ready to diff, since we requested
384
# content in 'topological' order
385
parent_keys = self.parent_map.pop(key)
386
# If a VersionedFile claims 'no-graph' support, then it may return
387
# None for any parent request, so we replace it with an empty tuple
388
if parent_keys is None:
391
for p in parent_keys:
392
# Alternatively we could check p not in self.needed_keys, but
393
# ghost_parents should be tiny versus huge
394
if p in self.ghost_parents:
396
refcount = self.refcounts[p]
397
if refcount == 1: # Last child reference
398
self.refcounts.pop(p)
399
parent_chunks = self.chunks.pop(p)
401
self.refcounts[p] = refcount - 1
402
parent_chunks = self.chunks[p]
403
p_lines = osutils.chunks_to_lines(parent_chunks)
404
# TODO: Should we cache the line form? We did the
405
# computation to get it, but storing it this way will
406
# be less memory efficient...
407
parent_lines.append(p_lines)
409
lines = osutils.chunks_to_lines(this_chunks)
410
# Since we needed the lines, we'll go ahead and cache them this way
412
self._compute_diff(key, parent_lines, lines)
414
# Is this content required for any more children?
415
if key in self.refcounts:
416
self.chunks[key] = this_chunks
418
def _extract_diffs(self):
419
needed_keys, refcounts = self._find_needed_keys()
420
for record in self.vf.get_record_stream(needed_keys,
421
'topological', True):
422
if record.storage_kind == 'absent':
423
raise errors.RevisionNotPresent(record.key, self.vf)
424
self._process_one_record(record.key,
425
record.get_bytes_as('chunked'))
427
def compute_diffs(self):
428
self._extract_diffs()
429
dpop = self.diffs.pop
430
return [dpop(k) for k in self.ordered_keys]
433
209
class VersionedFile(object):
434
210
"""Versioned text file storage.
525
301
self._check_write_ok()
526
302
return self._add_lines(version_id, parents, lines, parent_texts,
527
left_matching_blocks, nostore_sha, random_id, check_content)
303
left_matching_blocks, nostore_sha, random_id, check_content)
529
305
def _add_lines(self, version_id, parents, lines, parent_texts,
530
left_matching_blocks, nostore_sha, random_id, check_content):
306
left_matching_blocks, nostore_sha, random_id, check_content):
531
307
"""Helper to do the class specific add_lines."""
532
308
raise NotImplementedError(self.add_lines)
534
310
def add_lines_with_ghosts(self, version_id, parents, lines,
535
parent_texts=None, nostore_sha=None, random_id=False,
536
check_content=True, left_matching_blocks=None):
311
parent_texts=None, nostore_sha=None, random_id=False,
312
check_content=True, left_matching_blocks=None):
537
313
"""Add lines to the versioned file, allowing ghosts to be present.
539
315
This takes the same parameters as add_lines and returns the same.
541
317
self._check_write_ok()
542
318
return self._add_lines_with_ghosts(version_id, parents, lines,
543
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
319
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
545
321
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
546
nostore_sha, random_id, check_content, left_matching_blocks):
322
nostore_sha, random_id, check_content, left_matching_blocks):
547
323
"""Helper to do class specific add_lines_with_ghosts."""
548
324
raise NotImplementedError(self.add_lines_with_ghosts)
629
401
for version, parent_ids, expected_sha1, mpdiff in records:
630
402
needed_parents.update(p for p in parent_ids
631
403
if not mpvf.has_version(p))
632
present_parents = set(self.get_parent_map(needed_parents))
404
present_parents = set(self.get_parent_map(needed_parents).keys())
633
405
for parent_id, lines in zip(present_parents,
634
self._get_lf_split_line_list(present_parents)):
406
self._get_lf_split_line_list(present_parents)):
635
407
mpvf.add_version(lines, parent_id, [])
636
for (version, parent_ids, expected_sha1, mpdiff), lines in zip(
637
records, mpvf.get_line_list(versions)):
408
for (version, parent_ids, expected_sha1, mpdiff), lines in\
409
zip(records, mpvf.get_line_list(versions)):
638
410
if len(parent_ids) == 1:
639
411
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
640
mpvf.get_diff(parent_ids[0]).num_lines()))
412
mpvf.get_diff(parent_ids[0]).num_lines()))
642
414
left_matching_blocks = None
644
416
_, _, version_text = self.add_lines_with_ghosts(version,
645
parent_ids, lines, vf_parents,
646
left_matching_blocks=left_matching_blocks)
417
parent_ids, lines, vf_parents,
418
left_matching_blocks=left_matching_blocks)
647
419
except NotImplementedError:
648
420
# The vf can't handle ghosts, so add lines normally, which will
649
421
# (reasonably) fail if there are ghosts in the data.
650
422
_, _, version_text = self.add_lines(version,
651
parent_ids, lines, vf_parents,
652
left_matching_blocks=left_matching_blocks)
423
parent_ids, lines, vf_parents,
424
left_matching_blocks=left_matching_blocks)
653
425
vf_parents[version] = version_text
654
426
sha1s = self.get_sha1s(versions)
655
427
for version, parent_ids, expected_sha1, mpdiff in records:
682
454
raise NotImplementedError(self.get_lines)
684
456
def _get_lf_split_line_list(self, version_ids):
685
return [BytesIO(t).readlines() for t in self.get_texts(version_ids)]
457
return [StringIO(t).readlines() for t in self.get_texts(version_ids)]
687
def get_ancestry(self, version_ids):
459
def get_ancestry(self, version_ids, topo_sorted=True):
688
460
"""Return a list of all ancestors of given version(s). This
689
461
will not include the null revision.
463
This list will not be topologically sorted if topo_sorted=False is
691
466
Must raise RevisionNotPresent if any of the given versions are
692
467
not present in file history."""
468
if isinstance(version_ids, basestring):
469
version_ids = [version_ids]
693
470
raise NotImplementedError(self.get_ancestry)
695
472
def get_ancestry_with_ghosts(self, version_ids):
803
580
def add_lines(self, key, parents, lines, parent_texts=None,
804
left_matching_blocks=None, nostore_sha=None, random_id=False,
581
left_matching_blocks=None, nostore_sha=None, random_id=False,
806
583
self.calls.append(("add_lines", key, parents, lines, parent_texts,
807
left_matching_blocks, nostore_sha, random_id, check_content))
584
left_matching_blocks, nostore_sha, random_id, check_content))
808
585
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
809
left_matching_blocks, nostore_sha, random_id, check_content)
811
def add_content(self, factory, parent_texts=None,
812
left_matching_blocks=None, nostore_sha=None, random_id=False,
814
self.calls.append(("add_content", factory, parent_texts,
815
left_matching_blocks, nostore_sha, random_id, check_content))
816
return self._backing_vf.add_content(
817
factory, parent_texts, left_matching_blocks, nostore_sha,
818
random_id, check_content)
586
left_matching_blocks, nostore_sha, random_id, check_content)
821
589
self._backing_vf.check()
976
744
This mapper is for use with a transport based backend.
979
_safe = bytearray(b"abcdefghijklmnopqrstuvwxyz0123456789-_@,.")
747
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
981
749
def _escape(self, prefix):
982
750
"""Turn a key element into a filesystem safe string.
984
This is similar to a plain urlutils.quote, except
752
This is similar to a plain urllib.quote, except
985
753
it uses specific safe characters, so that it doesn't
986
754
have to translate a lot of valid file ids.
988
756
# @ does not get escaped. This is because it is a valid
989
757
# filesystem character we use all the time, and it looks
990
758
# a lot better than seeing %40 all the time.
991
r = [(c in self._safe) and chr(c) or ('%%%02x' % c)
992
for c in bytearray(prefix)]
993
return ''.join(r).encode('ascii')
759
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
995
763
def _unescape(self, basename):
996
764
"""Escaped names are easily unescaped by urlutils."""
997
return urlutils.unquote(basename)
765
return urllib.unquote(basename)
1000
768
def make_versioned_files_factory(versioned_file_factory, mapper):
1022
790
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
1023
791
may have a different length key-size, but that size will be constant for
1024
all texts added to or retrieved from it. For instance, breezy uses
792
all texts added to or retrieved from it. For instance, bzrlib uses
1025
793
instances with a key-size of 2 for storing user files in a repository, with
1026
794
the first element the fileid, and the second the version of that file.
1028
796
The use of tuples allows a single code base to support several different
1029
797
uses with only the mapping logic changing from instance to instance.
1031
:ivar _immediate_fallback_vfs: For subclasses that support stacking,
1032
this is a list of other VersionedFiles immediately underneath this
1033
one. They may in turn each have further fallbacks.
1036
800
def add_lines(self, key, parents, lines, parent_texts=None,
1037
left_matching_blocks=None, nostore_sha=None, random_id=False,
1038
check_content=True):
801
left_matching_blocks=None, nostore_sha=None, random_id=False,
1039
803
"""Add a text to the store.
1041
805
:param key: The key tuple of the text to add. If the last element is
1073
837
raise NotImplementedError(self.add_lines)
1075
def add_content(self, factory, parent_texts=None,
1076
left_matching_blocks=None, nostore_sha=None, random_id=False,
1077
check_content=True):
1078
"""Add a text to the store from a chunk iterable.
839
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
840
"""Add a text to the store.
842
This is a private function for use by CommitBuilder.
1080
844
:param key: The key tuple of the text to add. If the last element is
1081
845
None, a CHK string will be generated during the addition.
1082
846
:param parents: The parents key tuples of the text to add.
1083
:param chunk_iter: An iterable over bytestrings.
1084
:param parent_texts: An optional dictionary containing the opaque
1085
representations of some or all of the parents of version_id to
1086
allow delta optimisations. VERY IMPORTANT: the texts must be those
1087
returned by add_lines or data corruption can be caused.
1088
:param left_matching_blocks: a hint about which areas are common
1089
between the text and its left-hand-parent. The format is
1090
the SequenceMatcher.get_matching_blocks format.
847
:param text: A string containing the text to be committed.
1091
848
:param nostore_sha: Raise ExistingContent and do not add the lines to
1092
849
the versioned file if the digest of the lines matches this.
1093
850
:param random_id: If True a random id has been selected rather than
1122
884
if not mpvf.has_version(p))
1123
885
# It seems likely that adding all the present parents as fulltexts can
1124
886
# easily exhaust memory.
887
chunks_to_lines = osutils.chunks_to_lines
1125
888
for record in self.get_record_stream(needed_parents, 'unordered',
1127
890
if record.storage_kind == 'absent':
1129
mpvf.add_version(record.get_bytes_as('lines'), record.key, [])
1130
for (key, parent_keys, expected_sha1, mpdiff), lines in zip(
1131
records, mpvf.get_line_list(versions)):
892
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
894
for (key, parent_keys, expected_sha1, mpdiff), lines in\
895
zip(records, mpvf.get_line_list(versions)):
1132
896
if len(parent_keys) == 1:
1133
897
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
1134
mpvf.get_diff(parent_keys[0]).num_lines()))
898
mpvf.get_diff(parent_keys[0]).num_lines()))
1136
900
left_matching_blocks = None
1137
901
version_sha1, _, version_text = self.add_lines(key,
1138
parent_keys, lines, vf_parents,
1139
left_matching_blocks=left_matching_blocks)
902
parent_keys, lines, vf_parents,
903
left_matching_blocks=left_matching_blocks)
1140
904
if version_sha1 != expected_sha1:
1141
905
raise errors.VersionedFileInvalidChecksum(version)
1142
906
vf_parents[key] = version_text
1284
1048
def make_mpdiffs(self, keys):
1285
1049
"""Create multiparent diffs for specified keys."""
1286
generator = _MPDiffGenerator(self, keys)
1287
return generator.compute_diffs()
1289
def get_annotator(self):
1290
return annotate.Annotator(self)
1050
keys_order = tuple(keys)
1051
keys = frozenset(keys)
1052
knit_keys = set(keys)
1053
parent_map = self.get_parent_map(keys)
1054
for parent_keys in parent_map.itervalues():
1056
knit_keys.update(parent_keys)
1057
missing_keys = keys - set(parent_map)
1059
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1060
# We need to filter out ghosts, because we can't diff against them.
1061
maybe_ghosts = knit_keys - keys
1062
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1063
knit_keys.difference_update(ghosts)
1065
chunks_to_lines = osutils.chunks_to_lines
1066
for record in self.get_record_stream(knit_keys, 'topological', True):
1067
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1068
# line_block_dict = {}
1069
# for parent, blocks in record.extract_line_blocks():
1070
# line_blocks[parent] = blocks
1071
# line_blocks[record.key] = line_block_dict
1073
for key in keys_order:
1075
parents = parent_map[key] or []
1076
# Note that filtering knit_keys can lead to a parent difference
1077
# between the creation and the application of the mpdiff.
1078
parent_lines = [lines[p] for p in parents if p in knit_keys]
1079
if len(parent_lines) > 0:
1080
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1083
left_parent_blocks = None
1084
diffs.append(multiparent.MultiParent.from_lines(target,
1085
parent_lines, left_parent_blocks))
1292
1088
missing_keys = index._missing_keys_from_parent_map
1294
1090
def _extract_blocks(self, version_id, source, target):
1297
def _transitive_fallbacks(self):
1298
"""Return the whole stack of fallback versionedfiles.
1300
This VersionedFiles may have a list of fallbacks, but it doesn't
1301
necessarily know about the whole stack going down, and it can't know
1302
at open time because they may change after the objects are opened.
1305
for a_vfs in self._immediate_fallback_vfs:
1306
all_fallbacks.append(a_vfs)
1307
all_fallbacks.extend(a_vfs._transitive_fallbacks())
1308
return all_fallbacks
1311
1094
class ThunkedVersionedFiles(VersionedFiles):
1312
1095
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1325
1108
self._mapper = mapper
1326
1109
self._is_locked = is_locked
1328
def add_content(self, factory, parent_texts=None,
1329
left_matching_blocks=None, nostore_sha=None, random_id=False):
1330
"""See VersionedFiles.add_content()."""
1331
lines = factory.get_bytes_as('lines')
1332
return self.add_lines(
1333
factory.key, factory.parents, lines,
1334
parent_texts=parent_texts,
1335
left_matching_blocks=left_matching_blocks,
1336
nostore_sha=nostore_sha,
1337
random_id=random_id,
1340
1111
def add_lines(self, key, parents, lines, parent_texts=None,
1341
left_matching_blocks=None, nostore_sha=None, random_id=False,
1342
check_content=True):
1112
left_matching_blocks=None, nostore_sha=None, random_id=False,
1113
check_content=True):
1343
1114
"""See VersionedFiles.add_lines()."""
1344
1115
path = self._mapper.map(key)
1345
1116
version_id = key[-1]
1350
1121
return vf.add_lines_with_ghosts(version_id, parents, lines,
1351
parent_texts=parent_texts,
1352
left_matching_blocks=left_matching_blocks,
1353
nostore_sha=nostore_sha, random_id=random_id,
1354
check_content=check_content)
1122
parent_texts=parent_texts,
1123
left_matching_blocks=left_matching_blocks,
1124
nostore_sha=nostore_sha, random_id=random_id,
1125
check_content=check_content)
1355
1126
except NotImplementedError:
1356
1127
return vf.add_lines(version_id, parents, lines,
1357
parent_texts=parent_texts,
1358
left_matching_blocks=left_matching_blocks,
1359
nostore_sha=nostore_sha, random_id=random_id,
1360
check_content=check_content)
1128
parent_texts=parent_texts,
1129
left_matching_blocks=left_matching_blocks,
1130
nostore_sha=nostore_sha, random_id=random_id,
1131
check_content=check_content)
1361
1132
except errors.NoSuchFile:
1362
1133
# parent directory may be missing, try again.
1363
1134
self._transport.mkdir(osutils.dirname(path))
1365
1136
return vf.add_lines_with_ghosts(version_id, parents, lines,
1366
parent_texts=parent_texts,
1367
left_matching_blocks=left_matching_blocks,
1368
nostore_sha=nostore_sha, random_id=random_id,
1369
check_content=check_content)
1137
parent_texts=parent_texts,
1138
left_matching_blocks=left_matching_blocks,
1139
nostore_sha=nostore_sha, random_id=random_id,
1140
check_content=check_content)
1370
1141
except NotImplementedError:
1371
1142
return vf.add_lines(version_id, parents, lines,
1372
parent_texts=parent_texts,
1373
left_matching_blocks=left_matching_blocks,
1374
nostore_sha=nostore_sha, random_id=random_id,
1375
check_content=check_content)
1143
parent_texts=parent_texts,
1144
left_matching_blocks=left_matching_blocks,
1145
nostore_sha=nostore_sha, random_id=random_id,
1146
check_content=check_content)
1377
1148
def annotate(self, key):
1378
1149
"""Return a list of (version-key, line) tuples for the text of key.
1540
class VersionedFilesWithFallbacks(VersionedFiles):
1542
def without_fallbacks(self):
1543
"""Return a clone of this object without any fallbacks configured."""
1544
raise NotImplementedError(self.without_fallbacks)
1546
def add_fallback_versioned_files(self, a_versioned_files):
1547
"""Add a source of texts for texts not present in this knit.
1549
:param a_versioned_files: A VersionedFiles object.
1551
raise NotImplementedError(self.add_fallback_versioned_files)
1553
def get_known_graph_ancestry(self, keys):
1554
"""Get a KnownGraph instance with the ancestry of keys."""
1555
parent_map, missing_keys = self._index.find_ancestry(keys)
1556
for fallback in self._transitive_fallbacks():
1557
if not missing_keys:
1559
(f_parent_map, f_missing_keys) = fallback._index.find_ancestry(
1561
parent_map.update(f_parent_map)
1562
missing_keys = f_missing_keys
1563
kg = _mod_graph.KnownGraph(parent_map)
1567
1314
class _PlanMergeVersionedFile(VersionedFiles):
1568
1315
"""A VersionedFile for uncommitted and committed texts.
1590
1337
# line data for locally held keys.
1591
1338
self._lines = {}
1592
1339
# key lookup providers
1593
self._providers = [_mod_graph.DictParentsProvider(self._parents)]
1340
self._providers = [DictParentsProvider(self._parents)]
1595
1342
def plan_merge(self, ver_a, ver_b, base=None):
1596
1343
"""See VersionedFile.plan_merge"""
1597
from ..merge import _PlanMerge
1344
from bzrlib.merge import _PlanMerge
1598
1345
if base is None:
1599
1346
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1600
old_plan = list(_PlanMerge(ver_a, base, self,
1601
(self._file_id,)).plan_merge())
1602
new_plan = list(_PlanMerge(ver_a, ver_b, self,
1603
(self._file_id,)).plan_merge())
1347
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1348
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1604
1349
return _PlanMerge._subtract_plans(old_plan, new_plan)
1606
1351
def plan_lca_merge(self, ver_a, ver_b, base=None):
1607
from ..merge import _PlanLCAMerge
1608
graph = _mod_graph.Graph(self)
1609
new_plan = _PlanLCAMerge(
1610
ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1352
from bzrlib.merge import _PlanLCAMerge
1354
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1611
1355
if base is None:
1612
1356
return new_plan
1613
old_plan = _PlanLCAMerge(
1614
ver_a, base, self, (self._file_id,), graph).plan_merge()
1357
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1615
1358
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1617
def add_content(self, factory):
1618
return self.add_lines(
1619
factory.key, factory.parents, factory.get_bytes_as('lines'))
1621
1360
def add_lines(self, key, parents, lines):
1622
1361
"""See VersionedFiles.add_lines
1624
1363
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1625
1364
are permitted. Only reserved ids are permitted.
1627
if not isinstance(key, tuple):
1366
if type(key) is not tuple:
1628
1367
raise TypeError(key)
1629
1368
if not revision.is_reserved_id(key[-1]):
1630
1369
raise ValueError('Only reserved ids may be used')
1891
class NoDupeAddLinesDecorator(object):
1892
"""Decorator for a VersionedFiles that skips doing an add_lines if the key
1896
def __init__(self, store):
1899
def add_lines(self, key, parents, lines, parent_texts=None,
1900
left_matching_blocks=None, nostore_sha=None, random_id=False,
1901
check_content=True):
1902
"""See VersionedFiles.add_lines.
1904
This implementation may return None as the third element of the return
1905
value when the original store wouldn't.
1908
raise NotImplementedError(
1909
"NoDupeAddLinesDecorator.add_lines does not implement the "
1910
"nostore_sha behaviour.")
1912
sha1 = osutils.sha_strings(lines)
1913
key = (b"sha1:" + sha1,)
1916
if key in self._store.get_parent_map([key]):
1917
# This key has already been inserted, so don't do it again.
1919
sha1 = osutils.sha_strings(lines)
1920
return sha1, sum(map(len, lines)), None
1921
return self._store.add_lines(key, parents, lines,
1922
parent_texts=parent_texts,
1923
left_matching_blocks=left_matching_blocks,
1924
nostore_sha=nostore_sha, random_id=random_id,
1925
check_content=check_content)
1927
def __getattr__(self, name):
1928
return getattr(self._store, name)
1931
1627
def network_bytes_to_kind_and_offset(network_bytes):
1932
1628
"""Strip of a record kind from the front of network_bytes.
1934
1630
:param network_bytes: The bytes of a record.
1935
1631
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1937
line_end = network_bytes.find(b'\n')
1938
storage_kind = network_bytes[:line_end].decode('ascii')
1633
line_end = network_bytes.find('\n')
1634
storage_kind = network_bytes[:line_end]
1939
1635
return storage_kind, line_end + 1
1968
1664
for bytes in self._bytes_iterator:
1969
1665
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1970
1666
for record in self._kind_factory[storage_kind](
1971
storage_kind, bytes, line_end):
1667
storage_kind, bytes, line_end):
1975
1671
def fulltext_network_to_record(kind, bytes, line_end):
1976
1672
"""Convert a network fulltext record to record."""
1977
meta_len, = struct.unpack('!L', bytes[line_end:line_end + 4])
1978
record_meta = bytes[line_end + 4:line_end + 4 + meta_len]
1673
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1674
record_meta = bytes[line_end+4:line_end+4+meta_len]
1979
1675
key, parents = bencode.bdecode_as_tuple(record_meta)
1980
if parents == b'nil':
1676
if parents == 'nil':
1982
fulltext = bytes[line_end + 4 + meta_len:]
1678
fulltext = bytes[line_end+4+meta_len:]
1983
1679
return [FulltextContentFactory(key, parents, None, fulltext)]
2024
1720
for prefix in sorted(per_prefix_map):
2025
1721
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
2026
1722
return present_keys
2029
class _KeyRefs(object):
2031
def __init__(self, track_new_keys=False):
2032
# dict mapping 'key' to 'set of keys referring to that key'
2035
# set remembering all new keys
2036
self.new_keys = set()
2038
self.new_keys = None
2044
self.new_keys.clear()
2046
def add_references(self, key, refs):
2047
# Record the new references
2048
for referenced in refs:
2050
needed_by = self.refs[referenced]
2052
needed_by = self.refs[referenced] = set()
2054
# Discard references satisfied by the new key
2057
def get_new_keys(self):
2058
return self.new_keys
2060
def get_unsatisfied_refs(self):
2061
return self.refs.keys()
2063
def _satisfy_refs_for_key(self, key):
2067
# No keys depended on this key. That's ok.
2070
def add_key(self, key):
2071
# satisfy refs for key, and remember that we've seen this key.
2072
self._satisfy_refs_for_key(key)
2073
if self.new_keys is not None:
2074
self.new_keys.add(key)
2076
def satisfy_refs_for_keys(self, keys):
2078
self._satisfy_refs_for_key(key)
2080
def get_referrers(self):
2081
return set(itertools.chain.from_iterable(viewvalues(self.refs)))
added
removed
bzrlib/versionedfile.py
