20
17
"""Versioned text file storage api."""
19
from __future__ import absolute_import
22
21
from copy import copy
23
from cStringIO import StringIO
26
25
from zlib import adler32
28
from bzrlib.lazy_import import lazy_import
27
from ..lazy_import import lazy_import
29
28
lazy_import(globals(), """
35
33
graph as _mod_graph,
45
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
46
from bzrlib.transport.memory import MemoryTransport
40
from breezy.bzr import (
48
from bzrlib.registry import Registry
49
from bzrlib.symbol_versioning import *
50
from bzrlib.textmerge import TextMerge
51
from bzrlib import bencode
46
from ..registry import Registry
47
from ..sixish import (
53
from ..textmerge import TextMerge
54
56
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
56
'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
57
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'breezy.bzr.knit',
58
'DeltaPlainToFullText')
59
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'breezy.bzr.knit',
59
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
62
'breezy.bzr.knit', 'DeltaAnnotatedToUnannotated')
61
63
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
'bzrlib.knit', 'DeltaAnnotatedToFullText')
64
'breezy.bzr.knit', 'DeltaAnnotatedToFullText')
63
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
64
'bzrlib.knit', 'FTAnnotatedToUnannotated')
66
'breezy.bzr.knit', 'FTAnnotatedToUnannotated')
65
67
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
'bzrlib.knit', 'FTAnnotatedToFullText')
68
'breezy.bzr.knit', 'FTAnnotatedToFullText')
67
69
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
70
# 'breezy.bzr.knit', 'FTAnnotatedToChunked')
71
73
class ContentFactory(object):
213
class _MPDiffGenerator(object):
214
"""Pull out the functionality for generating mp_diffs."""
216
def __init__(self, vf, keys):
218
# This is the order the keys were requested in
219
self.ordered_keys = tuple(keys)
220
# keys + their parents, what we need to compute the diffs
221
self.needed_keys = ()
222
# Map from key: mp_diff
224
# Map from key: parents_needed (may have ghosts)
226
# Parents that aren't present
227
self.ghost_parents = ()
228
# Map from parent_key => number of children for this text
230
# Content chunks that are cached while we still need them
233
def _find_needed_keys(self):
234
"""Find the set of keys we need to request.
236
This includes all the original keys passed in, and the non-ghost
237
parents of those keys.
239
:return: (needed_keys, refcounts)
240
needed_keys is the set of all texts we need to extract
241
refcounts is a dict of {key: num_children} letting us know when we
242
no longer need to cache a given parent text
244
# All the keys and their parents
245
needed_keys = set(self.ordered_keys)
246
parent_map = self.vf.get_parent_map(needed_keys)
247
self.parent_map = parent_map
248
# TODO: Should we be using a different construct here? I think this
249
# uses difference_update internally, and we expect the result to
251
missing_keys = needed_keys.difference(parent_map)
253
raise errors.RevisionNotPresent(list(missing_keys)[0], self.vf)
254
# Parents that might be missing. They are allowed to be ghosts, but we
255
# should check for them
257
setdefault = refcounts.setdefault
259
for child_key, parent_keys in viewitems(parent_map):
261
# parent_keys may be None if a given VersionedFile claims to
262
# not support graph operations.
264
just_parents.update(parent_keys)
265
needed_keys.update(parent_keys)
266
for p in parent_keys:
267
refcounts[p] = setdefault(p, 0) + 1
268
just_parents.difference_update(parent_map)
269
# Remove any parents that are actually ghosts from the needed set
270
self.present_parents = set(self.vf.get_parent_map(just_parents))
271
self.ghost_parents = just_parents.difference(self.present_parents)
272
needed_keys.difference_update(self.ghost_parents)
273
self.needed_keys = needed_keys
274
self.refcounts = refcounts
275
return needed_keys, refcounts
277
def _compute_diff(self, key, parent_lines, lines):
278
"""Compute a single mp_diff, and store it in self._diffs"""
279
if len(parent_lines) > 0:
280
# XXX: _extract_blocks is not usefully defined anywhere...
281
# It was meant to extract the left-parent diff without
282
# having to recompute it for Knit content (pack-0.92,
283
# etc). That seems to have regressed somewhere
284
left_parent_blocks = self.vf._extract_blocks(key,
285
parent_lines[0], lines)
287
left_parent_blocks = None
288
diff = multiparent.MultiParent.from_lines(lines,
289
parent_lines, left_parent_blocks)
290
self.diffs[key] = diff
292
def _process_one_record(self, key, this_chunks):
294
if key in self.parent_map:
295
# This record should be ready to diff, since we requested
296
# content in 'topological' order
297
parent_keys = self.parent_map.pop(key)
298
# If a VersionedFile claims 'no-graph' support, then it may return
299
# None for any parent request, so we replace it with an empty tuple
300
if parent_keys is None:
303
for p in parent_keys:
304
# Alternatively we could check p not in self.needed_keys, but
305
# ghost_parents should be tiny versus huge
306
if p in self.ghost_parents:
308
refcount = self.refcounts[p]
309
if refcount == 1: # Last child reference
310
self.refcounts.pop(p)
311
parent_chunks = self.chunks.pop(p)
313
self.refcounts[p] = refcount - 1
314
parent_chunks = self.chunks[p]
315
p_lines = osutils.chunks_to_lines(parent_chunks)
316
# TODO: Should we cache the line form? We did the
317
# computation to get it, but storing it this way will
318
# be less memory efficient...
319
parent_lines.append(p_lines)
321
lines = osutils.chunks_to_lines(this_chunks)
322
# Since we needed the lines, we'll go ahead and cache them this way
324
self._compute_diff(key, parent_lines, lines)
326
# Is this content required for any more children?
327
if key in self.refcounts:
328
self.chunks[key] = this_chunks
330
def _extract_diffs(self):
331
needed_keys, refcounts = self._find_needed_keys()
332
for record in self.vf.get_record_stream(needed_keys,
333
'topological', True):
334
if record.storage_kind == 'absent':
335
raise errors.RevisionNotPresent(record.key, self.vf)
336
self._process_one_record(record.key,
337
record.get_bytes_as('chunked'))
339
def compute_diffs(self):
340
self._extract_diffs()
341
dpop = self.diffs.pop
342
return [dpop(k) for k in self.ordered_keys]
209
345
class VersionedFile(object):
210
346
"""Versioned text file storage.
301
437
self._check_write_ok()
302
438
return self._add_lines(version_id, parents, lines, parent_texts,
303
left_matching_blocks, nostore_sha, random_id, check_content)
439
left_matching_blocks, nostore_sha, random_id, check_content)
305
441
def _add_lines(self, version_id, parents, lines, parent_texts,
306
left_matching_blocks, nostore_sha, random_id, check_content):
442
left_matching_blocks, nostore_sha, random_id, check_content):
307
443
"""Helper to do the class specific add_lines."""
308
444
raise NotImplementedError(self.add_lines)
310
446
def add_lines_with_ghosts(self, version_id, parents, lines,
311
parent_texts=None, nostore_sha=None, random_id=False,
312
check_content=True, left_matching_blocks=None):
447
parent_texts=None, nostore_sha=None, random_id=False,
448
check_content=True, left_matching_blocks=None):
313
449
"""Add lines to the versioned file, allowing ghosts to be present.
315
451
This takes the same parameters as add_lines and returns the same.
317
453
self._check_write_ok()
318
454
return self._add_lines_with_ghosts(version_id, parents, lines,
319
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
455
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
321
457
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
322
nostore_sha, random_id, check_content, left_matching_blocks):
458
nostore_sha, random_id, check_content, left_matching_blocks):
323
459
"""Helper to do class specific add_lines_with_ghosts."""
324
460
raise NotImplementedError(self.add_lines_with_ghosts)
401
541
for version, parent_ids, expected_sha1, mpdiff in records:
402
542
needed_parents.update(p for p in parent_ids
403
543
if not mpvf.has_version(p))
404
present_parents = set(self.get_parent_map(needed_parents).keys())
544
present_parents = set(self.get_parent_map(needed_parents))
405
545
for parent_id, lines in zip(present_parents,
406
self._get_lf_split_line_list(present_parents)):
546
self._get_lf_split_line_list(present_parents)):
407
547
mpvf.add_version(lines, parent_id, [])
408
for (version, parent_ids, expected_sha1, mpdiff), lines in\
409
zip(records, mpvf.get_line_list(versions)):
548
for (version, parent_ids, expected_sha1, mpdiff), lines in zip(
549
records, mpvf.get_line_list(versions)):
410
550
if len(parent_ids) == 1:
411
551
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
412
mpvf.get_diff(parent_ids[0]).num_lines()))
552
mpvf.get_diff(parent_ids[0]).num_lines()))
414
554
left_matching_blocks = None
416
556
_, _, version_text = self.add_lines_with_ghosts(version,
417
parent_ids, lines, vf_parents,
418
left_matching_blocks=left_matching_blocks)
557
parent_ids, lines, vf_parents,
558
left_matching_blocks=left_matching_blocks)
419
559
except NotImplementedError:
420
560
# The vf can't handle ghosts, so add lines normally, which will
421
561
# (reasonably) fail if there are ghosts in the data.
422
562
_, _, version_text = self.add_lines(version,
423
parent_ids, lines, vf_parents,
424
left_matching_blocks=left_matching_blocks)
563
parent_ids, lines, vf_parents,
564
left_matching_blocks=left_matching_blocks)
425
565
vf_parents[version] = version_text
426
566
sha1s = self.get_sha1s(versions)
427
567
for version, parent_ids, expected_sha1, mpdiff in records:
580
718
def add_lines(self, key, parents, lines, parent_texts=None,
581
left_matching_blocks=None, nostore_sha=None, random_id=False,
719
left_matching_blocks=None, nostore_sha=None, random_id=False,
583
721
self.calls.append(("add_lines", key, parents, lines, parent_texts,
584
left_matching_blocks, nostore_sha, random_id, check_content))
722
left_matching_blocks, nostore_sha, random_id, check_content))
585
723
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
586
left_matching_blocks, nostore_sha, random_id, check_content)
724
left_matching_blocks, nostore_sha, random_id, check_content)
589
727
self._backing_vf.check()
744
882
This mapper is for use with a transport based backend.
747
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
885
_safe = bytearray(b"abcdefghijklmnopqrstuvwxyz0123456789-_@,.")
749
887
def _escape(self, prefix):
750
888
"""Turn a key element into a filesystem safe string.
752
This is similar to a plain urllib.quote, except
890
This is similar to a plain urlutils.quote, except
753
891
it uses specific safe characters, so that it doesn't
754
892
have to translate a lot of valid file ids.
756
894
# @ does not get escaped. This is because it is a valid
757
895
# filesystem character we use all the time, and it looks
758
896
# a lot better than seeing %40 all the time.
759
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
897
r = [(c in self._safe) and chr(c) or ('%%%02x' % c)
898
for c in bytearray(prefix)]
899
return ''.join(r).encode('ascii')
763
901
def _unescape(self, basename):
764
902
"""Escaped names are easily unescaped by urlutils."""
765
return urllib.unquote(basename)
903
return urlutils.unquote(basename)
768
906
def make_versioned_files_factory(versioned_file_factory, mapper):
790
928
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
791
929
may have a different length key-size, but that size will be constant for
792
all texts added to or retrieved from it. For instance, bzrlib uses
930
all texts added to or retrieved from it. For instance, breezy uses
793
931
instances with a key-size of 2 for storing user files in a repository, with
794
932
the first element the fileid, and the second the version of that file.
796
934
The use of tuples allows a single code base to support several different
797
935
uses with only the mapping logic changing from instance to instance.
937
:ivar _immediate_fallback_vfs: For subclasses that support stacking,
938
this is a list of other VersionedFiles immediately underneath this
939
one. They may in turn each have further fallbacks.
800
942
def add_lines(self, key, parents, lines, parent_texts=None,
801
left_matching_blocks=None, nostore_sha=None, random_id=False,
943
left_matching_blocks=None, nostore_sha=None, random_id=False,
803
945
"""Add a text to the store.
805
947
:param key: The key tuple of the text to add. If the last element is
837
979
raise NotImplementedError(self.add_lines)
839
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
840
"""Add a text to the store.
842
This is a private function for use by CommitBuilder.
981
def add_chunks(self, key, parents, chunk_iter, parent_texts=None,
982
left_matching_blocks=None, nostore_sha=None, random_id=False,
984
"""Add a text to the store from a chunk iterable.
844
986
:param key: The key tuple of the text to add. If the last element is
845
987
None, a CHK string will be generated during the addition.
846
988
:param parents: The parents key tuples of the text to add.
847
:param text: A string containing the text to be committed.
989
:param chunk_iter: An iterable over bytestrings.
990
:param parent_texts: An optional dictionary containing the opaque
991
representations of some or all of the parents of version_id to
992
allow delta optimisations. VERY IMPORTANT: the texts must be those
993
returned by add_lines or data corruption can be caused.
994
:param left_matching_blocks: a hint about which areas are common
995
between the text and its left-hand-parent. The format is
996
the SequenceMatcher.get_matching_blocks format.
848
997
:param nostore_sha: Raise ExistingContent and do not add the lines to
849
998
the versioned file if the digest of the lines matches this.
850
999
:param random_id: If True a random id has been selected rather than
886
1030
# easily exhaust memory.
887
1031
chunks_to_lines = osutils.chunks_to_lines
888
1032
for record in self.get_record_stream(needed_parents, 'unordered',
890
1034
if record.storage_kind == 'absent':
892
1036
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
894
for (key, parent_keys, expected_sha1, mpdiff), lines in\
895
zip(records, mpvf.get_line_list(versions)):
1038
for (key, parent_keys, expected_sha1, mpdiff), lines in zip(
1039
records, mpvf.get_line_list(versions)):
896
1040
if len(parent_keys) == 1:
897
1041
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
898
mpvf.get_diff(parent_keys[0]).num_lines()))
1042
mpvf.get_diff(parent_keys[0]).num_lines()))
900
1044
left_matching_blocks = None
901
1045
version_sha1, _, version_text = self.add_lines(key,
902
parent_keys, lines, vf_parents,
903
left_matching_blocks=left_matching_blocks)
1046
parent_keys, lines, vf_parents,
1047
left_matching_blocks=left_matching_blocks)
904
1048
if version_sha1 != expected_sha1:
905
1049
raise errors.VersionedFileInvalidChecksum(version)
906
1050
vf_parents[key] = version_text
1048
1192
def make_mpdiffs(self, keys):
1049
1193
"""Create multiparent diffs for specified keys."""
1050
keys_order = tuple(keys)
1051
keys = frozenset(keys)
1052
knit_keys = set(keys)
1053
parent_map = self.get_parent_map(keys)
1054
for parent_keys in parent_map.itervalues():
1056
knit_keys.update(parent_keys)
1057
missing_keys = keys - set(parent_map)
1059
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1060
# We need to filter out ghosts, because we can't diff against them.
1061
maybe_ghosts = knit_keys - keys
1062
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1063
knit_keys.difference_update(ghosts)
1065
chunks_to_lines = osutils.chunks_to_lines
1066
for record in self.get_record_stream(knit_keys, 'topological', True):
1067
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1068
# line_block_dict = {}
1069
# for parent, blocks in record.extract_line_blocks():
1070
# line_blocks[parent] = blocks
1071
# line_blocks[record.key] = line_block_dict
1073
for key in keys_order:
1075
parents = parent_map[key] or []
1076
# Note that filtering knit_keys can lead to a parent difference
1077
# between the creation and the application of the mpdiff.
1078
parent_lines = [lines[p] for p in parents if p in knit_keys]
1079
if len(parent_lines) > 0:
1080
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1083
left_parent_blocks = None
1084
diffs.append(multiparent.MultiParent.from_lines(target,
1085
parent_lines, left_parent_blocks))
1194
generator = _MPDiffGenerator(self, keys)
1195
return generator.compute_diffs()
1197
def get_annotator(self):
1198
return annotate.Annotator(self)
1088
1200
missing_keys = index._missing_keys_from_parent_map
1090
1202
def _extract_blocks(self, version_id, source, target):
1205
def _transitive_fallbacks(self):
1206
"""Return the whole stack of fallback versionedfiles.
1208
This VersionedFiles may have a list of fallbacks, but it doesn't
1209
necessarily know about the whole stack going down, and it can't know
1210
at open time because they may change after the objects are opened.
1213
for a_vfs in self._immediate_fallback_vfs:
1214
all_fallbacks.append(a_vfs)
1215
all_fallbacks.extend(a_vfs._transitive_fallbacks())
1216
return all_fallbacks
1094
1219
class ThunkedVersionedFiles(VersionedFiles):
1095
1220
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1108
1233
self._mapper = mapper
1109
1234
self._is_locked = is_locked
1236
def add_chunks(self, key, parents, chunk_iter, parent_texts=None,
1237
left_matching_blocks=None, nostore_sha=None,
1239
# For now, just fallback to add_lines.
1240
lines = osutils.chunks_to_lines(list(chunk_iter))
1241
return self.add_lines(
1242
key, parents, lines, parent_texts,
1243
left_matching_blocks, nostore_sha, random_id,
1111
1246
def add_lines(self, key, parents, lines, parent_texts=None,
1112
left_matching_blocks=None, nostore_sha=None, random_id=False,
1113
check_content=True):
1247
left_matching_blocks=None, nostore_sha=None, random_id=False,
1248
check_content=True):
1114
1249
"""See VersionedFiles.add_lines()."""
1115
1250
path = self._mapper.map(key)
1116
1251
version_id = key[-1]
1121
1256
return vf.add_lines_with_ghosts(version_id, parents, lines,
1122
parent_texts=parent_texts,
1123
left_matching_blocks=left_matching_blocks,
1124
nostore_sha=nostore_sha, random_id=random_id,
1125
check_content=check_content)
1257
parent_texts=parent_texts,
1258
left_matching_blocks=left_matching_blocks,
1259
nostore_sha=nostore_sha, random_id=random_id,
1260
check_content=check_content)
1126
1261
except NotImplementedError:
1127
1262
return vf.add_lines(version_id, parents, lines,
1128
parent_texts=parent_texts,
1129
left_matching_blocks=left_matching_blocks,
1130
nostore_sha=nostore_sha, random_id=random_id,
1131
check_content=check_content)
1263
parent_texts=parent_texts,
1264
left_matching_blocks=left_matching_blocks,
1265
nostore_sha=nostore_sha, random_id=random_id,
1266
check_content=check_content)
1132
1267
except errors.NoSuchFile:
1133
1268
# parent directory may be missing, try again.
1134
1269
self._transport.mkdir(osutils.dirname(path))
1136
1271
return vf.add_lines_with_ghosts(version_id, parents, lines,
1137
parent_texts=parent_texts,
1138
left_matching_blocks=left_matching_blocks,
1139
nostore_sha=nostore_sha, random_id=random_id,
1140
check_content=check_content)
1272
parent_texts=parent_texts,
1273
left_matching_blocks=left_matching_blocks,
1274
nostore_sha=nostore_sha, random_id=random_id,
1275
check_content=check_content)
1141
1276
except NotImplementedError:
1142
1277
return vf.add_lines(version_id, parents, lines,
1143
parent_texts=parent_texts,
1144
left_matching_blocks=left_matching_blocks,
1145
nostore_sha=nostore_sha, random_id=random_id,
1146
check_content=check_content)
1278
parent_texts=parent_texts,
1279
left_matching_blocks=left_matching_blocks,
1280
nostore_sha=nostore_sha, random_id=random_id,
1281
check_content=check_content)
1148
1283
def annotate(self, key):
1149
1284
"""Return a list of (version-key, line) tuples for the text of key.
1446
class VersionedFilesWithFallbacks(VersionedFiles):
1448
def without_fallbacks(self):
1449
"""Return a clone of this object without any fallbacks configured."""
1450
raise NotImplementedError(self.without_fallbacks)
1452
def add_fallback_versioned_files(self, a_versioned_files):
1453
"""Add a source of texts for texts not present in this knit.
1455
:param a_versioned_files: A VersionedFiles object.
1457
raise NotImplementedError(self.add_fallback_versioned_files)
1459
def get_known_graph_ancestry(self, keys):
1460
"""Get a KnownGraph instance with the ancestry of keys."""
1461
parent_map, missing_keys = self._index.find_ancestry(keys)
1462
for fallback in self._transitive_fallbacks():
1463
if not missing_keys:
1465
(f_parent_map, f_missing_keys) = fallback._index.find_ancestry(
1467
parent_map.update(f_parent_map)
1468
missing_keys = f_missing_keys
1469
kg = _mod_graph.KnownGraph(parent_map)
1314
1473
class _PlanMergeVersionedFile(VersionedFiles):
1315
1474
"""A VersionedFile for uncommitted and committed texts.
1337
1496
# line data for locally held keys.
1338
1497
self._lines = {}
1339
1498
# key lookup providers
1340
self._providers = [DictParentsProvider(self._parents)]
1499
self._providers = [_mod_graph.DictParentsProvider(self._parents)]
1342
1501
def plan_merge(self, ver_a, ver_b, base=None):
1343
1502
"""See VersionedFile.plan_merge"""
1344
from bzrlib.merge import _PlanMerge
1503
from ..merge import _PlanMerge
1345
1504
if base is None:
1346
1505
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1347
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1348
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1506
old_plan = list(_PlanMerge(ver_a, base, self,
1507
(self._file_id,)).plan_merge())
1508
new_plan = list(_PlanMerge(ver_a, ver_b, self,
1509
(self._file_id,)).plan_merge())
1349
1510
return _PlanMerge._subtract_plans(old_plan, new_plan)
1351
1512
def plan_lca_merge(self, ver_a, ver_b, base=None):
1352
from bzrlib.merge import _PlanLCAMerge
1354
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1513
from ..merge import _PlanLCAMerge
1514
graph = _mod_graph.Graph(self)
1515
new_plan = _PlanLCAMerge(
1516
ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1355
1517
if base is None:
1356
1518
return new_plan
1357
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1519
old_plan = _PlanLCAMerge(
1520
ver_a, base, self, (self._file_id,), graph).plan_merge()
1358
1521
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1360
1523
def add_lines(self, key, parents, lines):
1791
class NoDupeAddLinesDecorator(object):
1792
"""Decorator for a VersionedFiles that skips doing an add_lines if the key
1796
def __init__(self, store):
1799
def add_lines(self, key, parents, lines, parent_texts=None,
1800
left_matching_blocks=None, nostore_sha=None, random_id=False,
1801
check_content=True):
1802
"""See VersionedFiles.add_lines.
1804
This implementation may return None as the third element of the return
1805
value when the original store wouldn't.
1808
raise NotImplementedError(
1809
"NoDupeAddLinesDecorator.add_lines does not implement the "
1810
"nostore_sha behaviour.")
1812
sha1 = osutils.sha_strings(lines)
1813
key = (b"sha1:" + sha1,)
1816
if key in self._store.get_parent_map([key]):
1817
# This key has already been inserted, so don't do it again.
1819
sha1 = osutils.sha_strings(lines)
1820
return sha1, sum(map(len, lines)), None
1821
return self._store.add_lines(key, parents, lines,
1822
parent_texts=parent_texts,
1823
left_matching_blocks=left_matching_blocks,
1824
nostore_sha=nostore_sha, random_id=random_id,
1825
check_content=check_content)
1827
def __getattr__(self, name):
1828
return getattr(self._store, name)
1627
1831
def network_bytes_to_kind_and_offset(network_bytes):
1628
1832
"""Strip of a record kind from the front of network_bytes.
1630
1834
:param network_bytes: The bytes of a record.
1631
1835
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1633
line_end = network_bytes.find('\n')
1634
storage_kind = network_bytes[:line_end]
1837
line_end = network_bytes.find(b'\n')
1838
storage_kind = network_bytes[:line_end].decode('ascii')
1635
1839
return storage_kind, line_end + 1
1664
1868
for bytes in self._bytes_iterator:
1665
1869
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1666
1870
for record in self._kind_factory[storage_kind](
1667
storage_kind, bytes, line_end):
1871
storage_kind, bytes, line_end):
1671
1875
def fulltext_network_to_record(kind, bytes, line_end):
1672
1876
"""Convert a network fulltext record to record."""
1673
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1674
record_meta = bytes[line_end+4:line_end+4+meta_len]
1877
meta_len, = struct.unpack('!L', bytes[line_end:line_end + 4])
1878
record_meta = bytes[line_end + 4:line_end + 4 + meta_len]
1675
1879
key, parents = bencode.bdecode_as_tuple(record_meta)
1676
if parents == 'nil':
1880
if parents == b'nil':
1678
fulltext = bytes[line_end+4+meta_len:]
1882
fulltext = bytes[line_end + 4 + meta_len:]
1679
1883
return [FulltextContentFactory(key, parents, None, fulltext)]
1720
1924
for prefix in sorted(per_prefix_map):
1721
1925
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
1722
1926
return present_keys
1929
class _KeyRefs(object):
1931
def __init__(self, track_new_keys=False):
1932
# dict mapping 'key' to 'set of keys referring to that key'
1935
# set remembering all new keys
1936
self.new_keys = set()
1938
self.new_keys = None
1944
self.new_keys.clear()
1946
def add_references(self, key, refs):
1947
# Record the new references
1948
for referenced in refs:
1950
needed_by = self.refs[referenced]
1952
needed_by = self.refs[referenced] = set()
1954
# Discard references satisfied by the new key
1957
def get_new_keys(self):
1958
return self.new_keys
1960
def get_unsatisfied_refs(self):
1961
return self.refs.keys()
1963
def _satisfy_refs_for_key(self, key):
1967
# No keys depended on this key. That's ok.
1970
def add_key(self, key):
1971
# satisfy refs for key, and remember that we've seen this key.
1972
self._satisfy_refs_for_key(key)
1973
if self.new_keys is not None:
1974
self.new_keys.add(key)
1976
def satisfy_refs_for_keys(self, keys):
1978
self._satisfy_refs_for_key(key)
1980
def get_referrers(self):
1981
return set(itertools.chain.from_iterable(viewvalues(self.refs)))
added
removed
breezy/bzr/versionedfile.py
