20
17
"""Versioned text file storage api."""
22
19
from copy import copy
23
from cStringIO import StringIO
20
from io import BytesIO
26
24
from zlib import adler32
28
from bzrlib.lazy_import import lazy_import
26
from ..lazy_import import lazy_import
29
27
lazy_import(globals(), """
35
31
graph as _mod_graph,
45
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
46
from bzrlib.transport.memory import MemoryTransport
38
from breezy.bzr import (
48
from bzrlib.registry import Registry
49
from bzrlib.symbol_versioning import *
50
from bzrlib.textmerge import TextMerge
51
from bzrlib import bencode
47
from ..registry import Registry
48
from ..textmerge import TextMerge
54
51
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
56
'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
59
52
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
'bzrlib.knit', 'DeltaAnnotatedToFullText')
53
'breezy.bzr.knit', 'DeltaAnnotatedToUnannotated')
63
54
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
64
'bzrlib.knit', 'FTAnnotatedToUnannotated')
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
'bzrlib.knit', 'FTAnnotatedToFullText')
67
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
55
'breezy.bzr.knit', 'FTAnnotatedToUnannotated')
56
for target_storage_kind in ('fulltext', 'chunked', 'lines'):
57
adapter_registry.register_lazy(('knit-delta-gz', target_storage_kind), 'breezy.bzr.knit',
58
'DeltaPlainToFullText')
59
adapter_registry.register_lazy(('knit-ft-gz', target_storage_kind), 'breezy.bzr.knit',
61
adapter_registry.register_lazy(('knit-annotated-ft-gz', target_storage_kind),
62
'breezy.bzr.knit', 'FTAnnotatedToFullText')
63
adapter_registry.register_lazy(('knit-annotated-delta-gz', target_storage_kind),
64
'breezy.bzr.knit', 'DeltaAnnotatedToFullText')
71
67
class ContentFactory(object):
72
68
"""Abstract interface for insertion and retrieval from a VersionedFile.
74
70
:ivar sha1: None, or the sha1 of the content fulltext.
71
:ivar size: None, or the size of the content fulltext.
75
72
:ivar storage_kind: The native storage kind of this factory. One of
76
73
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
77
74
'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
106
105
:ivar parents: A tuple of parent keys for self.key. If the object has
107
106
no parent information, None (as opposed to () for an empty list of
108
:ivar chunks_are_lines: Whether chunks are lines.
111
def __init__(self, key, parents, sha1, chunks):
111
def __init__(self, key, parents, sha1, chunks, chunks_are_lines=None):
112
112
"""Create a ContentFactory."""
114
self.size = sum(map(len, chunks))
114
115
self.storage_kind = 'chunked'
116
117
self.parents = parents
117
118
self._chunks = chunks
119
self._chunks_are_lines = chunks_are_lines
119
121
def get_bytes_as(self, storage_kind):
120
122
if storage_kind == 'chunked':
121
123
return self._chunks
122
124
elif storage_kind == 'fulltext':
123
return ''.join(self._chunks)
125
return b''.join(self._chunks)
126
elif storage_kind == 'lines':
127
if self._chunks_are_lines:
129
return list(osutils.chunks_to_lines(self._chunks))
124
130
raise errors.UnavailableRepresentation(self.key, storage_kind,
133
def iter_bytes_as(self, storage_kind):
134
if storage_kind == 'chunked':
135
return iter(self._chunks)
136
elif storage_kind == 'lines':
137
if self._chunks_are_lines:
138
return iter(self._chunks)
139
return iter(osutils.chunks_to_lines(self._chunks))
140
raise errors.UnavailableRepresentation(self.key, storage_kind,
128
143
class FulltextContentFactory(ContentFactory):
129
144
"""Static data content factory.
154
172
return self._text
155
173
elif storage_kind == 'chunked':
156
174
return [self._text]
157
raise errors.UnavailableRepresentation(self.key, storage_kind,
175
elif storage_kind == 'lines':
176
return osutils.split_lines(self._text)
177
raise errors.UnavailableRepresentation(self.key, storage_kind,
180
def iter_bytes_as(self, storage_kind):
181
if storage_kind == 'chunked':
182
return iter([self._text])
183
elif storage_kind == 'lines':
184
return iter(osutils.split_lines(self._text))
185
raise errors.UnavailableRepresentation(self.key, storage_kind,
189
class FileContentFactory(ContentFactory):
190
"""File-based content factory.
193
def __init__(self, key, parents, fileobj, sha1=None, size=None):
195
self.parents = parents
197
self.storage_kind = 'file'
201
def get_bytes_as(self, storage_kind):
203
if storage_kind == 'fulltext':
204
return self.file.read()
205
elif storage_kind == 'chunked':
206
return list(osutils.file_iterator(self.file))
207
elif storage_kind == 'lines':
208
return list(self.file.readlines())
209
raise errors.UnavailableRepresentation(self.key, storage_kind,
212
def iter_bytes_as(self, storage_kind):
214
if storage_kind == 'chunked':
215
return osutils.file_iterator(self.file)
216
elif storage_kind == 'lines':
218
raise errors.UnavailableRepresentation(self.key, storage_kind,
161
222
class AbsentContentFactory(ContentFactory):
277
class _MPDiffGenerator(object):
278
"""Pull out the functionality for generating mp_diffs."""
280
def __init__(self, vf, keys):
282
# This is the order the keys were requested in
283
self.ordered_keys = tuple(keys)
284
# keys + their parents, what we need to compute the diffs
285
self.needed_keys = ()
286
# Map from key: mp_diff
288
# Map from key: parents_needed (may have ghosts)
290
# Parents that aren't present
291
self.ghost_parents = ()
292
# Map from parent_key => number of children for this text
294
# Content chunks that are cached while we still need them
297
def _find_needed_keys(self):
298
"""Find the set of keys we need to request.
300
This includes all the original keys passed in, and the non-ghost
301
parents of those keys.
303
:return: (needed_keys, refcounts)
304
needed_keys is the set of all texts we need to extract
305
refcounts is a dict of {key: num_children} letting us know when we
306
no longer need to cache a given parent text
308
# All the keys and their parents
309
needed_keys = set(self.ordered_keys)
310
parent_map = self.vf.get_parent_map(needed_keys)
311
self.parent_map = parent_map
312
# TODO: Should we be using a different construct here? I think this
313
# uses difference_update internally, and we expect the result to
315
missing_keys = needed_keys.difference(parent_map)
317
raise errors.RevisionNotPresent(list(missing_keys)[0], self.vf)
318
# Parents that might be missing. They are allowed to be ghosts, but we
319
# should check for them
321
setdefault = refcounts.setdefault
323
for child_key, parent_keys in parent_map.items():
325
# parent_keys may be None if a given VersionedFile claims to
326
# not support graph operations.
328
just_parents.update(parent_keys)
329
needed_keys.update(parent_keys)
330
for p in parent_keys:
331
refcounts[p] = setdefault(p, 0) + 1
332
just_parents.difference_update(parent_map)
333
# Remove any parents that are actually ghosts from the needed set
334
self.present_parents = set(self.vf.get_parent_map(just_parents))
335
self.ghost_parents = just_parents.difference(self.present_parents)
336
needed_keys.difference_update(self.ghost_parents)
337
self.needed_keys = needed_keys
338
self.refcounts = refcounts
339
return needed_keys, refcounts
341
def _compute_diff(self, key, parent_lines, lines):
342
"""Compute a single mp_diff, and store it in self._diffs"""
343
if len(parent_lines) > 0:
344
# XXX: _extract_blocks is not usefully defined anywhere...
345
# It was meant to extract the left-parent diff without
346
# having to recompute it for Knit content (pack-0.92,
347
# etc). That seems to have regressed somewhere
348
left_parent_blocks = self.vf._extract_blocks(key,
349
parent_lines[0], lines)
351
left_parent_blocks = None
352
diff = multiparent.MultiParent.from_lines(lines,
353
parent_lines, left_parent_blocks)
354
self.diffs[key] = diff
356
def _process_one_record(self, key, this_chunks):
358
if key in self.parent_map:
359
# This record should be ready to diff, since we requested
360
# content in 'topological' order
361
parent_keys = self.parent_map.pop(key)
362
# If a VersionedFile claims 'no-graph' support, then it may return
363
# None for any parent request, so we replace it with an empty tuple
364
if parent_keys is None:
367
for p in parent_keys:
368
# Alternatively we could check p not in self.needed_keys, but
369
# ghost_parents should be tiny versus huge
370
if p in self.ghost_parents:
372
refcount = self.refcounts[p]
373
if refcount == 1: # Last child reference
374
self.refcounts.pop(p)
375
parent_chunks = self.chunks.pop(p)
377
self.refcounts[p] = refcount - 1
378
parent_chunks = self.chunks[p]
379
p_lines = osutils.chunks_to_lines(parent_chunks)
380
# TODO: Should we cache the line form? We did the
381
# computation to get it, but storing it this way will
382
# be less memory efficient...
383
parent_lines.append(p_lines)
385
lines = osutils.chunks_to_lines(this_chunks)
386
# Since we needed the lines, we'll go ahead and cache them this way
388
self._compute_diff(key, parent_lines, lines)
390
# Is this content required for any more children?
391
if key in self.refcounts:
392
self.chunks[key] = this_chunks
394
def _extract_diffs(self):
395
needed_keys, refcounts = self._find_needed_keys()
396
for record in self.vf.get_record_stream(needed_keys,
397
'topological', True):
398
if record.storage_kind == 'absent':
399
raise errors.RevisionNotPresent(record.key, self.vf)
400
self._process_one_record(record.key,
401
record.get_bytes_as('chunked'))
403
def compute_diffs(self):
404
self._extract_diffs()
405
dpop = self.diffs.pop
406
return [dpop(k) for k in self.ordered_keys]
209
409
class VersionedFile(object):
210
410
"""Versioned text file storage.
301
501
self._check_write_ok()
302
502
return self._add_lines(version_id, parents, lines, parent_texts,
303
left_matching_blocks, nostore_sha, random_id, check_content)
503
left_matching_blocks, nostore_sha, random_id, check_content)
305
505
def _add_lines(self, version_id, parents, lines, parent_texts,
306
left_matching_blocks, nostore_sha, random_id, check_content):
506
left_matching_blocks, nostore_sha, random_id, check_content):
307
507
"""Helper to do the class specific add_lines."""
308
508
raise NotImplementedError(self.add_lines)
310
510
def add_lines_with_ghosts(self, version_id, parents, lines,
311
parent_texts=None, nostore_sha=None, random_id=False,
312
check_content=True, left_matching_blocks=None):
511
parent_texts=None, nostore_sha=None, random_id=False,
512
check_content=True, left_matching_blocks=None):
313
513
"""Add lines to the versioned file, allowing ghosts to be present.
315
515
This takes the same parameters as add_lines and returns the same.
317
517
self._check_write_ok()
318
518
return self._add_lines_with_ghosts(version_id, parents, lines,
319
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
519
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
321
521
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
322
nostore_sha, random_id, check_content, left_matching_blocks):
522
nostore_sha, random_id, check_content, left_matching_blocks):
323
523
"""Helper to do class specific add_lines_with_ghosts."""
324
524
raise NotImplementedError(self.add_lines_with_ghosts)
401
605
for version, parent_ids, expected_sha1, mpdiff in records:
402
606
needed_parents.update(p for p in parent_ids
403
607
if not mpvf.has_version(p))
404
present_parents = set(self.get_parent_map(needed_parents).keys())
608
present_parents = set(self.get_parent_map(needed_parents))
405
609
for parent_id, lines in zip(present_parents,
406
self._get_lf_split_line_list(present_parents)):
610
self._get_lf_split_line_list(present_parents)):
407
611
mpvf.add_version(lines, parent_id, [])
408
for (version, parent_ids, expected_sha1, mpdiff), lines in\
409
zip(records, mpvf.get_line_list(versions)):
612
for (version, parent_ids, expected_sha1, mpdiff), lines in zip(
613
records, mpvf.get_line_list(versions)):
410
614
if len(parent_ids) == 1:
411
615
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
412
mpvf.get_diff(parent_ids[0]).num_lines()))
616
mpvf.get_diff(parent_ids[0]).num_lines()))
414
618
left_matching_blocks = None
416
620
_, _, version_text = self.add_lines_with_ghosts(version,
417
parent_ids, lines, vf_parents,
418
left_matching_blocks=left_matching_blocks)
621
parent_ids, lines, vf_parents,
622
left_matching_blocks=left_matching_blocks)
419
623
except NotImplementedError:
420
624
# The vf can't handle ghosts, so add lines normally, which will
421
625
# (reasonably) fail if there are ghosts in the data.
422
626
_, _, version_text = self.add_lines(version,
423
parent_ids, lines, vf_parents,
424
left_matching_blocks=left_matching_blocks)
627
parent_ids, lines, vf_parents,
628
left_matching_blocks=left_matching_blocks)
425
629
vf_parents[version] = version_text
426
630
sha1s = self.get_sha1s(versions)
427
631
for version, parent_ids, expected_sha1, mpdiff in records:
580
782
def add_lines(self, key, parents, lines, parent_texts=None,
581
left_matching_blocks=None, nostore_sha=None, random_id=False,
783
left_matching_blocks=None, nostore_sha=None, random_id=False,
583
785
self.calls.append(("add_lines", key, parents, lines, parent_texts,
584
left_matching_blocks, nostore_sha, random_id, check_content))
786
left_matching_blocks, nostore_sha, random_id, check_content))
585
787
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
586
left_matching_blocks, nostore_sha, random_id, check_content)
788
left_matching_blocks, nostore_sha, random_id, check_content)
790
def add_content(self, factory, parent_texts=None,
791
left_matching_blocks=None, nostore_sha=None, random_id=False,
793
self.calls.append(("add_content", factory, parent_texts,
794
left_matching_blocks, nostore_sha, random_id, check_content))
795
return self._backing_vf.add_content(
796
factory, parent_texts, left_matching_blocks, nostore_sha,
797
random_id, check_content)
589
800
self._backing_vf.check()
744
955
This mapper is for use with a transport based backend.
747
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
958
_safe = bytearray(b"abcdefghijklmnopqrstuvwxyz0123456789-_@,.")
749
960
def _escape(self, prefix):
750
961
"""Turn a key element into a filesystem safe string.
752
This is similar to a plain urllib.quote, except
963
This is similar to a plain urlutils.quote, except
753
964
it uses specific safe characters, so that it doesn't
754
965
have to translate a lot of valid file ids.
756
967
# @ does not get escaped. This is because it is a valid
757
968
# filesystem character we use all the time, and it looks
758
969
# a lot better than seeing %40 all the time.
759
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
970
r = [(c in self._safe) and chr(c) or ('%%%02x' % c)
971
for c in bytearray(prefix)]
972
return ''.join(r).encode('ascii')
763
974
def _unescape(self, basename):
764
975
"""Escaped names are easily unescaped by urlutils."""
765
return urllib.unquote(basename)
976
return urlutils.unquote(basename)
768
979
def make_versioned_files_factory(versioned_file_factory, mapper):
790
1001
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
791
1002
may have a different length key-size, but that size will be constant for
792
all texts added to or retrieved from it. For instance, bzrlib uses
1003
all texts added to or retrieved from it. For instance, breezy uses
793
1004
instances with a key-size of 2 for storing user files in a repository, with
794
1005
the first element the fileid, and the second the version of that file.
796
1007
The use of tuples allows a single code base to support several different
797
1008
uses with only the mapping logic changing from instance to instance.
1010
:ivar _immediate_fallback_vfs: For subclasses that support stacking,
1011
this is a list of other VersionedFiles immediately underneath this
1012
one. They may in turn each have further fallbacks.
800
1015
def add_lines(self, key, parents, lines, parent_texts=None,
801
left_matching_blocks=None, nostore_sha=None, random_id=False,
1016
left_matching_blocks=None, nostore_sha=None, random_id=False,
1017
check_content=True):
803
1018
"""Add a text to the store.
805
1020
:param key: The key tuple of the text to add. If the last element is
837
1052
raise NotImplementedError(self.add_lines)
839
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
840
"""Add a text to the store.
842
This is a private function for use by CommitBuilder.
1054
def add_content(self, factory, parent_texts=None,
1055
left_matching_blocks=None, nostore_sha=None, random_id=False,
1056
check_content=True):
1057
"""Add a text to the store from a chunk iterable.
844
1059
:param key: The key tuple of the text to add. If the last element is
845
1060
None, a CHK string will be generated during the addition.
846
1061
:param parents: The parents key tuples of the text to add.
847
:param text: A string containing the text to be committed.
1062
:param chunk_iter: An iterable over bytestrings.
1063
:param parent_texts: An optional dictionary containing the opaque
1064
representations of some or all of the parents of version_id to
1065
allow delta optimisations. VERY IMPORTANT: the texts must be those
1066
returned by add_lines or data corruption can be caused.
1067
:param left_matching_blocks: a hint about which areas are common
1068
between the text and its left-hand-parent. The format is
1069
the SequenceMatcher.get_matching_blocks format.
848
1070
:param nostore_sha: Raise ExistingContent and do not add the lines to
849
1071
the versioned file if the digest of the lines matches this.
850
1072
:param random_id: If True a random id has been selected rather than
884
1101
if not mpvf.has_version(p))
885
1102
# It seems likely that adding all the present parents as fulltexts can
886
1103
# easily exhaust memory.
887
chunks_to_lines = osutils.chunks_to_lines
888
1104
for record in self.get_record_stream(needed_parents, 'unordered',
890
1106
if record.storage_kind == 'absent':
892
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
894
for (key, parent_keys, expected_sha1, mpdiff), lines in\
895
zip(records, mpvf.get_line_list(versions)):
1108
mpvf.add_version(record.get_bytes_as('lines'), record.key, [])
1109
for (key, parent_keys, expected_sha1, mpdiff), lines in zip(
1110
records, mpvf.get_line_list(versions)):
896
1111
if len(parent_keys) == 1:
897
1112
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
898
mpvf.get_diff(parent_keys[0]).num_lines()))
1113
mpvf.get_diff(parent_keys[0]).num_lines()))
900
1115
left_matching_blocks = None
901
1116
version_sha1, _, version_text = self.add_lines(key,
902
parent_keys, lines, vf_parents,
903
left_matching_blocks=left_matching_blocks)
1117
parent_keys, lines, vf_parents,
1118
left_matching_blocks=left_matching_blocks)
904
1119
if version_sha1 != expected_sha1:
905
1120
raise errors.VersionedFileInvalidChecksum(version)
906
1121
vf_parents[key] = version_text
1048
1263
def make_mpdiffs(self, keys):
1049
1264
"""Create multiparent diffs for specified keys."""
1050
keys_order = tuple(keys)
1051
keys = frozenset(keys)
1052
knit_keys = set(keys)
1053
parent_map = self.get_parent_map(keys)
1054
for parent_keys in parent_map.itervalues():
1056
knit_keys.update(parent_keys)
1057
missing_keys = keys - set(parent_map)
1059
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1060
# We need to filter out ghosts, because we can't diff against them.
1061
maybe_ghosts = knit_keys - keys
1062
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1063
knit_keys.difference_update(ghosts)
1065
chunks_to_lines = osutils.chunks_to_lines
1066
for record in self.get_record_stream(knit_keys, 'topological', True):
1067
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1068
# line_block_dict = {}
1069
# for parent, blocks in record.extract_line_blocks():
1070
# line_blocks[parent] = blocks
1071
# line_blocks[record.key] = line_block_dict
1073
for key in keys_order:
1075
parents = parent_map[key] or []
1076
# Note that filtering knit_keys can lead to a parent difference
1077
# between the creation and the application of the mpdiff.
1078
parent_lines = [lines[p] for p in parents if p in knit_keys]
1079
if len(parent_lines) > 0:
1080
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1083
left_parent_blocks = None
1084
diffs.append(multiparent.MultiParent.from_lines(target,
1085
parent_lines, left_parent_blocks))
1265
generator = _MPDiffGenerator(self, keys)
1266
return generator.compute_diffs()
1268
def get_annotator(self):
1269
return annotate.Annotator(self)
1088
1271
missing_keys = index._missing_keys_from_parent_map
1090
1273
def _extract_blocks(self, version_id, source, target):
1276
def _transitive_fallbacks(self):
1277
"""Return the whole stack of fallback versionedfiles.
1279
This VersionedFiles may have a list of fallbacks, but it doesn't
1280
necessarily know about the whole stack going down, and it can't know
1281
at open time because they may change after the objects are opened.
1284
for a_vfs in self._immediate_fallback_vfs:
1285
all_fallbacks.append(a_vfs)
1286
all_fallbacks.extend(a_vfs._transitive_fallbacks())
1287
return all_fallbacks
1094
1290
class ThunkedVersionedFiles(VersionedFiles):
1095
1291
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1108
1304
self._mapper = mapper
1109
1305
self._is_locked = is_locked
1307
def add_content(self, factory, parent_texts=None,
1308
left_matching_blocks=None, nostore_sha=None, random_id=False):
1309
"""See VersionedFiles.add_content()."""
1310
lines = factory.get_bytes_as('lines')
1311
return self.add_lines(
1312
factory.key, factory.parents, lines,
1313
parent_texts=parent_texts,
1314
left_matching_blocks=left_matching_blocks,
1315
nostore_sha=nostore_sha,
1316
random_id=random_id,
1111
1319
def add_lines(self, key, parents, lines, parent_texts=None,
1112
left_matching_blocks=None, nostore_sha=None, random_id=False,
1113
check_content=True):
1320
left_matching_blocks=None, nostore_sha=None, random_id=False,
1321
check_content=True):
1114
1322
"""See VersionedFiles.add_lines()."""
1115
1323
path = self._mapper.map(key)
1116
1324
version_id = key[-1]
1121
1329
return vf.add_lines_with_ghosts(version_id, parents, lines,
1122
parent_texts=parent_texts,
1123
left_matching_blocks=left_matching_blocks,
1124
nostore_sha=nostore_sha, random_id=random_id,
1125
check_content=check_content)
1330
parent_texts=parent_texts,
1331
left_matching_blocks=left_matching_blocks,
1332
nostore_sha=nostore_sha, random_id=random_id,
1333
check_content=check_content)
1126
1334
except NotImplementedError:
1127
1335
return vf.add_lines(version_id, parents, lines,
1128
parent_texts=parent_texts,
1129
left_matching_blocks=left_matching_blocks,
1130
nostore_sha=nostore_sha, random_id=random_id,
1131
check_content=check_content)
1336
parent_texts=parent_texts,
1337
left_matching_blocks=left_matching_blocks,
1338
nostore_sha=nostore_sha, random_id=random_id,
1339
check_content=check_content)
1132
1340
except errors.NoSuchFile:
1133
1341
# parent directory may be missing, try again.
1134
1342
self._transport.mkdir(osutils.dirname(path))
1136
1344
return vf.add_lines_with_ghosts(version_id, parents, lines,
1137
parent_texts=parent_texts,
1138
left_matching_blocks=left_matching_blocks,
1139
nostore_sha=nostore_sha, random_id=random_id,
1140
check_content=check_content)
1345
parent_texts=parent_texts,
1346
left_matching_blocks=left_matching_blocks,
1347
nostore_sha=nostore_sha, random_id=random_id,
1348
check_content=check_content)
1141
1349
except NotImplementedError:
1142
1350
return vf.add_lines(version_id, parents, lines,
1143
parent_texts=parent_texts,
1144
left_matching_blocks=left_matching_blocks,
1145
nostore_sha=nostore_sha, random_id=random_id,
1146
check_content=check_content)
1351
parent_texts=parent_texts,
1352
left_matching_blocks=left_matching_blocks,
1353
nostore_sha=nostore_sha, random_id=random_id,
1354
check_content=check_content)
1148
1356
def annotate(self, key):
1149
1357
"""Return a list of (version-key, line) tuples for the text of key.
1519
class VersionedFilesWithFallbacks(VersionedFiles):
1521
def without_fallbacks(self):
1522
"""Return a clone of this object without any fallbacks configured."""
1523
raise NotImplementedError(self.without_fallbacks)
1525
def add_fallback_versioned_files(self, a_versioned_files):
1526
"""Add a source of texts for texts not present in this knit.
1528
:param a_versioned_files: A VersionedFiles object.
1530
raise NotImplementedError(self.add_fallback_versioned_files)
1532
def get_known_graph_ancestry(self, keys):
1533
"""Get a KnownGraph instance with the ancestry of keys."""
1534
parent_map, missing_keys = self._index.find_ancestry(keys)
1535
for fallback in self._transitive_fallbacks():
1536
if not missing_keys:
1538
(f_parent_map, f_missing_keys) = fallback._index.find_ancestry(
1540
parent_map.update(f_parent_map)
1541
missing_keys = f_missing_keys
1542
kg = _mod_graph.KnownGraph(parent_map)
1314
1546
class _PlanMergeVersionedFile(VersionedFiles):
1315
1547
"""A VersionedFile for uncommitted and committed texts.
1337
1569
# line data for locally held keys.
1338
1570
self._lines = {}
1339
1571
# key lookup providers
1340
self._providers = [DictParentsProvider(self._parents)]
1572
self._providers = [_mod_graph.DictParentsProvider(self._parents)]
1342
1574
def plan_merge(self, ver_a, ver_b, base=None):
1343
1575
"""See VersionedFile.plan_merge"""
1344
from bzrlib.merge import _PlanMerge
1576
from ..merge import _PlanMerge
1345
1577
if base is None:
1346
1578
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1347
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1348
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1579
old_plan = list(_PlanMerge(ver_a, base, self,
1580
(self._file_id,)).plan_merge())
1581
new_plan = list(_PlanMerge(ver_a, ver_b, self,
1582
(self._file_id,)).plan_merge())
1349
1583
return _PlanMerge._subtract_plans(old_plan, new_plan)
1351
1585
def plan_lca_merge(self, ver_a, ver_b, base=None):
1352
from bzrlib.merge import _PlanLCAMerge
1354
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1586
from ..merge import _PlanLCAMerge
1587
graph = _mod_graph.Graph(self)
1588
new_plan = _PlanLCAMerge(
1589
ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1355
1590
if base is None:
1356
1591
return new_plan
1357
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1592
old_plan = _PlanLCAMerge(
1593
ver_a, base, self, (self._file_id,), graph).plan_merge()
1358
1594
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1596
def add_content(self, factory):
1597
return self.add_lines(
1598
factory.key, factory.parents, factory.get_bytes_as('lines'))
1360
1600
def add_lines(self, key, parents, lines):
1361
1601
"""See VersionedFiles.add_lines
1363
1603
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1364
1604
are permitted. Only reserved ids are permitted.
1366
if type(key) is not tuple:
1606
if not isinstance(key, tuple):
1367
1607
raise TypeError(key)
1368
1608
if not revision.is_reserved_id(key[-1]):
1369
1609
raise ValueError('Only reserved ids may be used')
1870
class NoDupeAddLinesDecorator(object):
1871
"""Decorator for a VersionedFiles that skips doing an add_lines if the key
1875
def __init__(self, store):
1878
def add_lines(self, key, parents, lines, parent_texts=None,
1879
left_matching_blocks=None, nostore_sha=None, random_id=False,
1880
check_content=True):
1881
"""See VersionedFiles.add_lines.
1883
This implementation may return None as the third element of the return
1884
value when the original store wouldn't.
1887
raise NotImplementedError(
1888
"NoDupeAddLinesDecorator.add_lines does not implement the "
1889
"nostore_sha behaviour.")
1891
sha1 = osutils.sha_strings(lines)
1892
key = (b"sha1:" + sha1,)
1895
if key in self._store.get_parent_map([key]):
1896
# This key has already been inserted, so don't do it again.
1898
sha1 = osutils.sha_strings(lines)
1899
return sha1, sum(map(len, lines)), None
1900
return self._store.add_lines(key, parents, lines,
1901
parent_texts=parent_texts,
1902
left_matching_blocks=left_matching_blocks,
1903
nostore_sha=nostore_sha, random_id=random_id,
1904
check_content=check_content)
1906
def __getattr__(self, name):
1907
return getattr(self._store, name)
1627
1910
def network_bytes_to_kind_and_offset(network_bytes):
1628
1911
"""Strip of a record kind from the front of network_bytes.
1630
1913
:param network_bytes: The bytes of a record.
1631
1914
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1633
line_end = network_bytes.find('\n')
1634
storage_kind = network_bytes[:line_end]
1916
line_end = network_bytes.find(b'\n')
1917
storage_kind = network_bytes[:line_end].decode('ascii')
1635
1918
return storage_kind, line_end + 1
1664
1947
for bytes in self._bytes_iterator:
1665
1948
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1666
1949
for record in self._kind_factory[storage_kind](
1667
storage_kind, bytes, line_end):
1950
storage_kind, bytes, line_end):
1671
1954
def fulltext_network_to_record(kind, bytes, line_end):
1672
1955
"""Convert a network fulltext record to record."""
1673
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1674
record_meta = bytes[line_end+4:line_end+4+meta_len]
1956
meta_len, = struct.unpack('!L', bytes[line_end:line_end + 4])
1957
record_meta = bytes[line_end + 4:line_end + 4 + meta_len]
1675
1958
key, parents = bencode.bdecode_as_tuple(record_meta)
1676
if parents == 'nil':
1959
if parents == b'nil':
1678
fulltext = bytes[line_end+4+meta_len:]
1961
fulltext = bytes[line_end + 4 + meta_len:]
1679
1962
return [FulltextContentFactory(key, parents, None, fulltext)]
1720
2003
for prefix in sorted(per_prefix_map):
1721
2004
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
1722
2005
return present_keys
2008
class _KeyRefs(object):
2010
def __init__(self, track_new_keys=False):
2011
# dict mapping 'key' to 'set of keys referring to that key'
2014
# set remembering all new keys
2015
self.new_keys = set()
2017
self.new_keys = None
2023
self.new_keys.clear()
2025
def add_references(self, key, refs):
2026
# Record the new references
2027
for referenced in refs:
2029
needed_by = self.refs[referenced]
2031
needed_by = self.refs[referenced] = set()
2033
# Discard references satisfied by the new key
2036
def get_new_keys(self):
2037
return self.new_keys
2039
def get_unsatisfied_refs(self):
2040
return self.refs.keys()
2042
def _satisfy_refs_for_key(self, key):
2046
# No keys depended on this key. That's ok.
2049
def add_key(self, key):
2050
# satisfy refs for key, and remember that we've seen this key.
2051
self._satisfy_refs_for_key(key)
2052
if self.new_keys is not None:
2053
self.new_keys.add(key)
2055
def satisfy_refs_for_keys(self, keys):
2057
self._satisfy_refs_for_key(key)
2059
def get_referrers(self):
2060
return set(itertools.chain.from_iterable(self.refs.values()))
added
removed
breezy/bzr/versionedfile.py
