17
20
"""Versioned text file storage api."""
19
from __future__ import absolute_import
21
22
from copy import copy
23
from cStringIO import StringIO
25
26
from zlib import adler32
27
from ..lazy_import import lazy_import
28
from bzrlib.lazy_import import lazy_import
28
29
lazy_import(globals(), """
32
35
graph as _mod_graph,
39
from breezy.bzr import (
45
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
46
from bzrlib.transport.memory import MemoryTransport
48
from ..registry import Registry
49
from ..sixish import (
55
from ..textmerge import TextMerge
48
from bzrlib.registry import Registry
49
from bzrlib.symbol_versioning import *
50
from bzrlib.textmerge import TextMerge
51
from bzrlib import bencode
58
54
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
56
'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
59
59
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
'breezy.bzr.knit', 'DeltaAnnotatedToUnannotated')
60
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
'bzrlib.knit', 'DeltaAnnotatedToFullText')
61
63
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
62
'breezy.bzr.knit', 'FTAnnotatedToUnannotated')
63
for target_storage_kind in ('fulltext', 'chunked', 'lines'):
64
adapter_registry.register_lazy(('knit-delta-gz', target_storage_kind), 'breezy.bzr.knit',
65
'DeltaPlainToFullText')
66
adapter_registry.register_lazy(('knit-ft-gz', target_storage_kind), 'breezy.bzr.knit',
68
adapter_registry.register_lazy(('knit-annotated-ft-gz', target_storage_kind),
69
'breezy.bzr.knit', 'FTAnnotatedToFullText')
70
adapter_registry.register_lazy(('knit-annotated-delta-gz', target_storage_kind),
71
'breezy.bzr.knit', 'DeltaAnnotatedToFullText')
64
'bzrlib.knit', 'FTAnnotatedToUnannotated')
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
'bzrlib.knit', 'FTAnnotatedToFullText')
67
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
74
71
class ContentFactory(object):
75
72
"""Abstract interface for insertion and retrieval from a VersionedFile.
77
74
:ivar sha1: None, or the sha1 of the content fulltext.
78
:ivar size: None, or the size of the content fulltext.
79
75
:ivar storage_kind: The native storage kind of this factory. One of
80
76
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
81
77
'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
112
106
:ivar parents: A tuple of parent keys for self.key. If the object has
113
107
no parent information, None (as opposed to () for an empty list of
115
:ivar chunks_are_lines: Whether chunks are lines.
118
def __init__(self, key, parents, sha1, chunks, chunks_are_lines=None):
111
def __init__(self, key, parents, sha1, chunks):
119
112
"""Create a ContentFactory."""
121
self.size = sum(map(len, chunks))
122
114
self.storage_kind = 'chunked'
124
116
self.parents = parents
125
117
self._chunks = chunks
126
self._chunks_are_lines = chunks_are_lines
128
119
def get_bytes_as(self, storage_kind):
129
120
if storage_kind == 'chunked':
130
121
return self._chunks
131
122
elif storage_kind == 'fulltext':
132
return b''.join(self._chunks)
133
elif storage_kind == 'lines':
134
if self._chunks_are_lines:
136
return list(osutils.chunks_to_lines(self._chunks))
123
return ''.join(self._chunks)
137
124
raise errors.UnavailableRepresentation(self.key, storage_kind,
140
def iter_bytes_as(self, storage_kind):
141
if storage_kind == 'chunked':
142
return iter(self._chunks)
143
elif storage_kind == 'lines':
144
if self._chunks_are_lines:
145
return iter(self._chunks)
146
return iter(osutils.chunks_to_lines(self._chunks))
147
raise errors.UnavailableRepresentation(self.key, storage_kind,
150
128
class FulltextContentFactory(ContentFactory):
151
129
"""Static data content factory.
179
154
return self._text
180
155
elif storage_kind == 'chunked':
181
156
return [self._text]
182
elif storage_kind == 'lines':
183
return osutils.split_lines(self._text)
184
raise errors.UnavailableRepresentation(self.key, storage_kind,
187
def iter_bytes_as(self, storage_kind):
188
if storage_kind == 'chunked':
189
return iter([self._text])
190
elif storage_kind == 'lines':
191
return iter(osutils.split_lines(self._text))
192
raise errors.UnavailableRepresentation(self.key, storage_kind,
196
class FileContentFactory(ContentFactory):
197
"""File-based content factory.
200
def __init__(self, key, parents, fileobj, sha1=None, size=None):
202
self.parents = parents
204
self.storage_kind = 'file'
208
def get_bytes_as(self, storage_kind):
210
if storage_kind == 'fulltext':
211
return self.file.read()
212
elif storage_kind == 'chunked':
213
return list(osutils.file_iterator(self.file))
214
elif storage_kind == 'lines':
215
return list(self.file.readlines())
216
raise errors.UnavailableRepresentation(self.key, storage_kind,
219
def iter_bytes_as(self, storage_kind):
221
if storage_kind == 'chunked':
222
return osutils.file_iterator(self.file)
223
elif storage_kind == 'lines':
225
raise errors.UnavailableRepresentation(self.key, storage_kind,
157
raise errors.UnavailableRepresentation(self.key, storage_kind,
229
161
class AbsentContentFactory(ContentFactory):
284
class _MPDiffGenerator(object):
285
"""Pull out the functionality for generating mp_diffs."""
287
def __init__(self, vf, keys):
289
# This is the order the keys were requested in
290
self.ordered_keys = tuple(keys)
291
# keys + their parents, what we need to compute the diffs
292
self.needed_keys = ()
293
# Map from key: mp_diff
295
# Map from key: parents_needed (may have ghosts)
297
# Parents that aren't present
298
self.ghost_parents = ()
299
# Map from parent_key => number of children for this text
301
# Content chunks that are cached while we still need them
304
def _find_needed_keys(self):
305
"""Find the set of keys we need to request.
307
This includes all the original keys passed in, and the non-ghost
308
parents of those keys.
310
:return: (needed_keys, refcounts)
311
needed_keys is the set of all texts we need to extract
312
refcounts is a dict of {key: num_children} letting us know when we
313
no longer need to cache a given parent text
315
# All the keys and their parents
316
needed_keys = set(self.ordered_keys)
317
parent_map = self.vf.get_parent_map(needed_keys)
318
self.parent_map = parent_map
319
# TODO: Should we be using a different construct here? I think this
320
# uses difference_update internally, and we expect the result to
322
missing_keys = needed_keys.difference(parent_map)
324
raise errors.RevisionNotPresent(list(missing_keys)[0], self.vf)
325
# Parents that might be missing. They are allowed to be ghosts, but we
326
# should check for them
328
setdefault = refcounts.setdefault
330
for child_key, parent_keys in viewitems(parent_map):
332
# parent_keys may be None if a given VersionedFile claims to
333
# not support graph operations.
335
just_parents.update(parent_keys)
336
needed_keys.update(parent_keys)
337
for p in parent_keys:
338
refcounts[p] = setdefault(p, 0) + 1
339
just_parents.difference_update(parent_map)
340
# Remove any parents that are actually ghosts from the needed set
341
self.present_parents = set(self.vf.get_parent_map(just_parents))
342
self.ghost_parents = just_parents.difference(self.present_parents)
343
needed_keys.difference_update(self.ghost_parents)
344
self.needed_keys = needed_keys
345
self.refcounts = refcounts
346
return needed_keys, refcounts
348
def _compute_diff(self, key, parent_lines, lines):
349
"""Compute a single mp_diff, and store it in self._diffs"""
350
if len(parent_lines) > 0:
351
# XXX: _extract_blocks is not usefully defined anywhere...
352
# It was meant to extract the left-parent diff without
353
# having to recompute it for Knit content (pack-0.92,
354
# etc). That seems to have regressed somewhere
355
left_parent_blocks = self.vf._extract_blocks(key,
356
parent_lines[0], lines)
358
left_parent_blocks = None
359
diff = multiparent.MultiParent.from_lines(lines,
360
parent_lines, left_parent_blocks)
361
self.diffs[key] = diff
363
def _process_one_record(self, key, this_chunks):
365
if key in self.parent_map:
366
# This record should be ready to diff, since we requested
367
# content in 'topological' order
368
parent_keys = self.parent_map.pop(key)
369
# If a VersionedFile claims 'no-graph' support, then it may return
370
# None for any parent request, so we replace it with an empty tuple
371
if parent_keys is None:
374
for p in parent_keys:
375
# Alternatively we could check p not in self.needed_keys, but
376
# ghost_parents should be tiny versus huge
377
if p in self.ghost_parents:
379
refcount = self.refcounts[p]
380
if refcount == 1: # Last child reference
381
self.refcounts.pop(p)
382
parent_chunks = self.chunks.pop(p)
384
self.refcounts[p] = refcount - 1
385
parent_chunks = self.chunks[p]
386
p_lines = osutils.chunks_to_lines(parent_chunks)
387
# TODO: Should we cache the line form? We did the
388
# computation to get it, but storing it this way will
389
# be less memory efficient...
390
parent_lines.append(p_lines)
392
lines = osutils.chunks_to_lines(this_chunks)
393
# Since we needed the lines, we'll go ahead and cache them this way
395
self._compute_diff(key, parent_lines, lines)
397
# Is this content required for any more children?
398
if key in self.refcounts:
399
self.chunks[key] = this_chunks
401
def _extract_diffs(self):
402
needed_keys, refcounts = self._find_needed_keys()
403
for record in self.vf.get_record_stream(needed_keys,
404
'topological', True):
405
if record.storage_kind == 'absent':
406
raise errors.RevisionNotPresent(record.key, self.vf)
407
self._process_one_record(record.key,
408
record.get_bytes_as('chunked'))
410
def compute_diffs(self):
411
self._extract_diffs()
412
dpop = self.diffs.pop
413
return [dpop(k) for k in self.ordered_keys]
416
209
class VersionedFile(object):
417
210
"""Versioned text file storage.
508
301
self._check_write_ok()
509
302
return self._add_lines(version_id, parents, lines, parent_texts,
510
left_matching_blocks, nostore_sha, random_id, check_content)
303
left_matching_blocks, nostore_sha, random_id, check_content)
512
305
def _add_lines(self, version_id, parents, lines, parent_texts,
513
left_matching_blocks, nostore_sha, random_id, check_content):
306
left_matching_blocks, nostore_sha, random_id, check_content):
514
307
"""Helper to do the class specific add_lines."""
515
308
raise NotImplementedError(self.add_lines)
517
310
def add_lines_with_ghosts(self, version_id, parents, lines,
518
parent_texts=None, nostore_sha=None, random_id=False,
519
check_content=True, left_matching_blocks=None):
311
parent_texts=None, nostore_sha=None, random_id=False,
312
check_content=True, left_matching_blocks=None):
520
313
"""Add lines to the versioned file, allowing ghosts to be present.
522
315
This takes the same parameters as add_lines and returns the same.
524
317
self._check_write_ok()
525
318
return self._add_lines_with_ghosts(version_id, parents, lines,
526
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
319
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
528
321
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
529
nostore_sha, random_id, check_content, left_matching_blocks):
322
nostore_sha, random_id, check_content, left_matching_blocks):
530
323
"""Helper to do class specific add_lines_with_ghosts."""
531
324
raise NotImplementedError(self.add_lines_with_ghosts)
612
401
for version, parent_ids, expected_sha1, mpdiff in records:
613
402
needed_parents.update(p for p in parent_ids
614
403
if not mpvf.has_version(p))
615
present_parents = set(self.get_parent_map(needed_parents))
404
present_parents = set(self.get_parent_map(needed_parents).keys())
616
405
for parent_id, lines in zip(present_parents,
617
self._get_lf_split_line_list(present_parents)):
406
self._get_lf_split_line_list(present_parents)):
618
407
mpvf.add_version(lines, parent_id, [])
619
for (version, parent_ids, expected_sha1, mpdiff), lines in zip(
620
records, mpvf.get_line_list(versions)):
408
for (version, parent_ids, expected_sha1, mpdiff), lines in\
409
zip(records, mpvf.get_line_list(versions)):
621
410
if len(parent_ids) == 1:
622
411
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
623
mpvf.get_diff(parent_ids[0]).num_lines()))
412
mpvf.get_diff(parent_ids[0]).num_lines()))
625
414
left_matching_blocks = None
627
416
_, _, version_text = self.add_lines_with_ghosts(version,
628
parent_ids, lines, vf_parents,
629
left_matching_blocks=left_matching_blocks)
417
parent_ids, lines, vf_parents,
418
left_matching_blocks=left_matching_blocks)
630
419
except NotImplementedError:
631
420
# The vf can't handle ghosts, so add lines normally, which will
632
421
# (reasonably) fail if there are ghosts in the data.
633
422
_, _, version_text = self.add_lines(version,
634
parent_ids, lines, vf_parents,
635
left_matching_blocks=left_matching_blocks)
423
parent_ids, lines, vf_parents,
424
left_matching_blocks=left_matching_blocks)
636
425
vf_parents[version] = version_text
637
426
sha1s = self.get_sha1s(versions)
638
427
for version, parent_ids, expected_sha1, mpdiff in records:
789
580
def add_lines(self, key, parents, lines, parent_texts=None,
790
left_matching_blocks=None, nostore_sha=None, random_id=False,
581
left_matching_blocks=None, nostore_sha=None, random_id=False,
792
583
self.calls.append(("add_lines", key, parents, lines, parent_texts,
793
left_matching_blocks, nostore_sha, random_id, check_content))
584
left_matching_blocks, nostore_sha, random_id, check_content))
794
585
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
795
left_matching_blocks, nostore_sha, random_id, check_content)
797
def add_content(self, factory, parent_texts=None,
798
left_matching_blocks=None, nostore_sha=None, random_id=False,
800
self.calls.append(("add_content", factory, parent_texts,
801
left_matching_blocks, nostore_sha, random_id, check_content))
802
return self._backing_vf.add_content(
803
factory, parent_texts, left_matching_blocks, nostore_sha,
804
random_id, check_content)
586
left_matching_blocks, nostore_sha, random_id, check_content)
807
589
self._backing_vf.check()
962
744
This mapper is for use with a transport based backend.
965
_safe = bytearray(b"abcdefghijklmnopqrstuvwxyz0123456789-_@,.")
747
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
967
749
def _escape(self, prefix):
968
750
"""Turn a key element into a filesystem safe string.
970
This is similar to a plain urlutils.quote, except
752
This is similar to a plain urllib.quote, except
971
753
it uses specific safe characters, so that it doesn't
972
754
have to translate a lot of valid file ids.
974
756
# @ does not get escaped. This is because it is a valid
975
757
# filesystem character we use all the time, and it looks
976
758
# a lot better than seeing %40 all the time.
977
r = [(c in self._safe) and chr(c) or ('%%%02x' % c)
978
for c in bytearray(prefix)]
979
return ''.join(r).encode('ascii')
759
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
981
763
def _unescape(self, basename):
982
764
"""Escaped names are easily unescaped by urlutils."""
983
return urlutils.unquote(basename)
765
return urllib.unquote(basename)
986
768
def make_versioned_files_factory(versioned_file_factory, mapper):
1008
790
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
1009
791
may have a different length key-size, but that size will be constant for
1010
all texts added to or retrieved from it. For instance, breezy uses
792
all texts added to or retrieved from it. For instance, bzrlib uses
1011
793
instances with a key-size of 2 for storing user files in a repository, with
1012
794
the first element the fileid, and the second the version of that file.
1014
796
The use of tuples allows a single code base to support several different
1015
797
uses with only the mapping logic changing from instance to instance.
1017
:ivar _immediate_fallback_vfs: For subclasses that support stacking,
1018
this is a list of other VersionedFiles immediately underneath this
1019
one. They may in turn each have further fallbacks.
1022
800
def add_lines(self, key, parents, lines, parent_texts=None,
1023
left_matching_blocks=None, nostore_sha=None, random_id=False,
1024
check_content=True):
801
left_matching_blocks=None, nostore_sha=None, random_id=False,
1025
803
"""Add a text to the store.
1027
805
:param key: The key tuple of the text to add. If the last element is
1059
837
raise NotImplementedError(self.add_lines)
1061
def add_content(self, factory, parent_texts=None,
1062
left_matching_blocks=None, nostore_sha=None, random_id=False,
1063
check_content=True):
1064
"""Add a text to the store from a chunk iterable.
839
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
840
"""Add a text to the store.
842
This is a private function for use by CommitBuilder.
1066
844
:param key: The key tuple of the text to add. If the last element is
1067
845
None, a CHK string will be generated during the addition.
1068
846
:param parents: The parents key tuples of the text to add.
1069
:param chunk_iter: An iterable over bytestrings.
1070
:param parent_texts: An optional dictionary containing the opaque
1071
representations of some or all of the parents of version_id to
1072
allow delta optimisations. VERY IMPORTANT: the texts must be those
1073
returned by add_lines or data corruption can be caused.
1074
:param left_matching_blocks: a hint about which areas are common
1075
between the text and its left-hand-parent. The format is
1076
the SequenceMatcher.get_matching_blocks format.
847
:param text: A string containing the text to be committed.
1077
848
:param nostore_sha: Raise ExistingContent and do not add the lines to
1078
849
the versioned file if the digest of the lines matches this.
1079
850
:param random_id: If True a random id has been selected rather than
1108
884
if not mpvf.has_version(p))
1109
885
# It seems likely that adding all the present parents as fulltexts can
1110
886
# easily exhaust memory.
887
chunks_to_lines = osutils.chunks_to_lines
1111
888
for record in self.get_record_stream(needed_parents, 'unordered',
1113
890
if record.storage_kind == 'absent':
1115
mpvf.add_version(record.get_bytes_as('lines'), record.key, [])
1116
for (key, parent_keys, expected_sha1, mpdiff), lines in zip(
1117
records, mpvf.get_line_list(versions)):
892
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
894
for (key, parent_keys, expected_sha1, mpdiff), lines in\
895
zip(records, mpvf.get_line_list(versions)):
1118
896
if len(parent_keys) == 1:
1119
897
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
1120
mpvf.get_diff(parent_keys[0]).num_lines()))
898
mpvf.get_diff(parent_keys[0]).num_lines()))
1122
900
left_matching_blocks = None
1123
901
version_sha1, _, version_text = self.add_lines(key,
1124
parent_keys, lines, vf_parents,
1125
left_matching_blocks=left_matching_blocks)
902
parent_keys, lines, vf_parents,
903
left_matching_blocks=left_matching_blocks)
1126
904
if version_sha1 != expected_sha1:
1127
905
raise errors.VersionedFileInvalidChecksum(version)
1128
906
vf_parents[key] = version_text
1270
1048
def make_mpdiffs(self, keys):
1271
1049
"""Create multiparent diffs for specified keys."""
1272
generator = _MPDiffGenerator(self, keys)
1273
return generator.compute_diffs()
1275
def get_annotator(self):
1276
return annotate.Annotator(self)
1050
keys_order = tuple(keys)
1051
keys = frozenset(keys)
1052
knit_keys = set(keys)
1053
parent_map = self.get_parent_map(keys)
1054
for parent_keys in parent_map.itervalues():
1056
knit_keys.update(parent_keys)
1057
missing_keys = keys - set(parent_map)
1059
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1060
# We need to filter out ghosts, because we can't diff against them.
1061
maybe_ghosts = knit_keys - keys
1062
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1063
knit_keys.difference_update(ghosts)
1065
chunks_to_lines = osutils.chunks_to_lines
1066
for record in self.get_record_stream(knit_keys, 'topological', True):
1067
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1068
# line_block_dict = {}
1069
# for parent, blocks in record.extract_line_blocks():
1070
# line_blocks[parent] = blocks
1071
# line_blocks[record.key] = line_block_dict
1073
for key in keys_order:
1075
parents = parent_map[key] or []
1076
# Note that filtering knit_keys can lead to a parent difference
1077
# between the creation and the application of the mpdiff.
1078
parent_lines = [lines[p] for p in parents if p in knit_keys]
1079
if len(parent_lines) > 0:
1080
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1083
left_parent_blocks = None
1084
diffs.append(multiparent.MultiParent.from_lines(target,
1085
parent_lines, left_parent_blocks))
1278
1088
missing_keys = index._missing_keys_from_parent_map
1280
1090
def _extract_blocks(self, version_id, source, target):
1283
def _transitive_fallbacks(self):
1284
"""Return the whole stack of fallback versionedfiles.
1286
This VersionedFiles may have a list of fallbacks, but it doesn't
1287
necessarily know about the whole stack going down, and it can't know
1288
at open time because they may change after the objects are opened.
1291
for a_vfs in self._immediate_fallback_vfs:
1292
all_fallbacks.append(a_vfs)
1293
all_fallbacks.extend(a_vfs._transitive_fallbacks())
1294
return all_fallbacks
1297
1094
class ThunkedVersionedFiles(VersionedFiles):
1298
1095
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1311
1108
self._mapper = mapper
1312
1109
self._is_locked = is_locked
1314
def add_content(self, factory, parent_texts=None,
1315
left_matching_blocks=None, nostore_sha=None, random_id=False):
1316
"""See VersionedFiles.add_content()."""
1317
lines = factory.get_bytes_as('lines')
1318
return self.add_lines(
1319
factory.key, factory.parents, lines,
1320
parent_texts=parent_texts,
1321
left_matching_blocks=left_matching_blocks,
1322
nostore_sha=nostore_sha,
1323
random_id=random_id,
1326
1111
def add_lines(self, key, parents, lines, parent_texts=None,
1327
left_matching_blocks=None, nostore_sha=None, random_id=False,
1328
check_content=True):
1112
left_matching_blocks=None, nostore_sha=None, random_id=False,
1113
check_content=True):
1329
1114
"""See VersionedFiles.add_lines()."""
1330
1115
path = self._mapper.map(key)
1331
1116
version_id = key[-1]
1336
1121
return vf.add_lines_with_ghosts(version_id, parents, lines,
1337
parent_texts=parent_texts,
1338
left_matching_blocks=left_matching_blocks,
1339
nostore_sha=nostore_sha, random_id=random_id,
1340
check_content=check_content)
1122
parent_texts=parent_texts,
1123
left_matching_blocks=left_matching_blocks,
1124
nostore_sha=nostore_sha, random_id=random_id,
1125
check_content=check_content)
1341
1126
except NotImplementedError:
1342
1127
return vf.add_lines(version_id, parents, lines,
1343
parent_texts=parent_texts,
1344
left_matching_blocks=left_matching_blocks,
1345
nostore_sha=nostore_sha, random_id=random_id,
1346
check_content=check_content)
1128
parent_texts=parent_texts,
1129
left_matching_blocks=left_matching_blocks,
1130
nostore_sha=nostore_sha, random_id=random_id,
1131
check_content=check_content)
1347
1132
except errors.NoSuchFile:
1348
1133
# parent directory may be missing, try again.
1349
1134
self._transport.mkdir(osutils.dirname(path))
1351
1136
return vf.add_lines_with_ghosts(version_id, parents, lines,
1352
parent_texts=parent_texts,
1353
left_matching_blocks=left_matching_blocks,
1354
nostore_sha=nostore_sha, random_id=random_id,
1355
check_content=check_content)
1137
parent_texts=parent_texts,
1138
left_matching_blocks=left_matching_blocks,
1139
nostore_sha=nostore_sha, random_id=random_id,
1140
check_content=check_content)
1356
1141
except NotImplementedError:
1357
1142
return vf.add_lines(version_id, parents, lines,
1358
parent_texts=parent_texts,
1359
left_matching_blocks=left_matching_blocks,
1360
nostore_sha=nostore_sha, random_id=random_id,
1361
check_content=check_content)
1143
parent_texts=parent_texts,
1144
left_matching_blocks=left_matching_blocks,
1145
nostore_sha=nostore_sha, random_id=random_id,
1146
check_content=check_content)
1363
1148
def annotate(self, key):
1364
1149
"""Return a list of (version-key, line) tuples for the text of key.
1526
class VersionedFilesWithFallbacks(VersionedFiles):
1528
def without_fallbacks(self):
1529
"""Return a clone of this object without any fallbacks configured."""
1530
raise NotImplementedError(self.without_fallbacks)
1532
def add_fallback_versioned_files(self, a_versioned_files):
1533
"""Add a source of texts for texts not present in this knit.
1535
:param a_versioned_files: A VersionedFiles object.
1537
raise NotImplementedError(self.add_fallback_versioned_files)
1539
def get_known_graph_ancestry(self, keys):
1540
"""Get a KnownGraph instance with the ancestry of keys."""
1541
parent_map, missing_keys = self._index.find_ancestry(keys)
1542
for fallback in self._transitive_fallbacks():
1543
if not missing_keys:
1545
(f_parent_map, f_missing_keys) = fallback._index.find_ancestry(
1547
parent_map.update(f_parent_map)
1548
missing_keys = f_missing_keys
1549
kg = _mod_graph.KnownGraph(parent_map)
1553
1314
class _PlanMergeVersionedFile(VersionedFiles):
1554
1315
"""A VersionedFile for uncommitted and committed texts.
1576
1337
# line data for locally held keys.
1577
1338
self._lines = {}
1578
1339
# key lookup providers
1579
self._providers = [_mod_graph.DictParentsProvider(self._parents)]
1340
self._providers = [DictParentsProvider(self._parents)]
1581
1342
def plan_merge(self, ver_a, ver_b, base=None):
1582
1343
"""See VersionedFile.plan_merge"""
1583
from ..merge import _PlanMerge
1344
from bzrlib.merge import _PlanMerge
1584
1345
if base is None:
1585
1346
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1586
old_plan = list(_PlanMerge(ver_a, base, self,
1587
(self._file_id,)).plan_merge())
1588
new_plan = list(_PlanMerge(ver_a, ver_b, self,
1589
(self._file_id,)).plan_merge())
1347
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1348
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1590
1349
return _PlanMerge._subtract_plans(old_plan, new_plan)
1592
1351
def plan_lca_merge(self, ver_a, ver_b, base=None):
1593
from ..merge import _PlanLCAMerge
1594
graph = _mod_graph.Graph(self)
1595
new_plan = _PlanLCAMerge(
1596
ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1352
from bzrlib.merge import _PlanLCAMerge
1354
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1597
1355
if base is None:
1598
1356
return new_plan
1599
old_plan = _PlanLCAMerge(
1600
ver_a, base, self, (self._file_id,), graph).plan_merge()
1357
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1601
1358
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1603
def add_content(self, factory):
1604
return self.add_lines(
1605
factory.key, factory.parents, factory.get_bytes_as('lines'))
1607
1360
def add_lines(self, key, parents, lines):
1608
1361
"""See VersionedFiles.add_lines
1610
1363
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1611
1364
are permitted. Only reserved ids are permitted.
1613
if not isinstance(key, tuple):
1366
if type(key) is not tuple:
1614
1367
raise TypeError(key)
1615
1368
if not revision.is_reserved_id(key[-1]):
1616
1369
raise ValueError('Only reserved ids may be used')
1877
class NoDupeAddLinesDecorator(object):
1878
"""Decorator for a VersionedFiles that skips doing an add_lines if the key
1882
def __init__(self, store):
1885
def add_lines(self, key, parents, lines, parent_texts=None,
1886
left_matching_blocks=None, nostore_sha=None, random_id=False,
1887
check_content=True):
1888
"""See VersionedFiles.add_lines.
1890
This implementation may return None as the third element of the return
1891
value when the original store wouldn't.
1894
raise NotImplementedError(
1895
"NoDupeAddLinesDecorator.add_lines does not implement the "
1896
"nostore_sha behaviour.")
1898
sha1 = osutils.sha_strings(lines)
1899
key = (b"sha1:" + sha1,)
1902
if key in self._store.get_parent_map([key]):
1903
# This key has already been inserted, so don't do it again.
1905
sha1 = osutils.sha_strings(lines)
1906
return sha1, sum(map(len, lines)), None
1907
return self._store.add_lines(key, parents, lines,
1908
parent_texts=parent_texts,
1909
left_matching_blocks=left_matching_blocks,
1910
nostore_sha=nostore_sha, random_id=random_id,
1911
check_content=check_content)
1913
def __getattr__(self, name):
1914
return getattr(self._store, name)
1917
1627
def network_bytes_to_kind_and_offset(network_bytes):
1918
1628
"""Strip of a record kind from the front of network_bytes.
1920
1630
:param network_bytes: The bytes of a record.
1921
1631
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1923
line_end = network_bytes.find(b'\n')
1924
storage_kind = network_bytes[:line_end].decode('ascii')
1633
line_end = network_bytes.find('\n')
1634
storage_kind = network_bytes[:line_end]
1925
1635
return storage_kind, line_end + 1
1954
1664
for bytes in self._bytes_iterator:
1955
1665
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1956
1666
for record in self._kind_factory[storage_kind](
1957
storage_kind, bytes, line_end):
1667
storage_kind, bytes, line_end):
1961
1671
def fulltext_network_to_record(kind, bytes, line_end):
1962
1672
"""Convert a network fulltext record to record."""
1963
meta_len, = struct.unpack('!L', bytes[line_end:line_end + 4])
1964
record_meta = bytes[line_end + 4:line_end + 4 + meta_len]
1673
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1674
record_meta = bytes[line_end+4:line_end+4+meta_len]
1965
1675
key, parents = bencode.bdecode_as_tuple(record_meta)
1966
if parents == b'nil':
1676
if parents == 'nil':
1968
fulltext = bytes[line_end + 4 + meta_len:]
1678
fulltext = bytes[line_end+4+meta_len:]
1969
1679
return [FulltextContentFactory(key, parents, None, fulltext)]
2010
1720
for prefix in sorted(per_prefix_map):
2011
1721
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
2012
1722
return present_keys
2015
class _KeyRefs(object):
2017
def __init__(self, track_new_keys=False):
2018
# dict mapping 'key' to 'set of keys referring to that key'
2021
# set remembering all new keys
2022
self.new_keys = set()
2024
self.new_keys = None
2030
self.new_keys.clear()
2032
def add_references(self, key, refs):
2033
# Record the new references
2034
for referenced in refs:
2036
needed_by = self.refs[referenced]
2038
needed_by = self.refs[referenced] = set()
2040
# Discard references satisfied by the new key
2043
def get_new_keys(self):
2044
return self.new_keys
2046
def get_unsatisfied_refs(self):
2047
return self.refs.keys()
2049
def _satisfy_refs_for_key(self, key):
2053
# No keys depended on this key. That's ok.
2056
def add_key(self, key):
2057
# satisfy refs for key, and remember that we've seen this key.
2058
self._satisfy_refs_for_key(key)
2059
if self.new_keys is not None:
2060
self.new_keys.add(key)
2062
def satisfy_refs_for_keys(self, keys):
2064
self._satisfy_refs_for_key(key)
2066
def get_referrers(self):
2067
return set(itertools.chain.from_iterable(viewvalues(self.refs)))
added
removed
bzrlib/versionedfile.py
