45
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
46
from bzrlib.transport.memory import MemoryTransport
48
from bzrlib.registry import Registry
49
from bzrlib.symbol_versioning import *
50
from bzrlib.textmerge import TextMerge
51
from bzrlib import bencode
43
from .registry import Registry
47
from .textmerge import TextMerge
54
50
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
51
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'breezy.knit',
56
52
'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
53
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'breezy.knit',
58
54
'FTPlainToFullText')
59
55
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
56
'breezy.knit', 'DeltaAnnotatedToUnannotated')
61
57
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
'bzrlib.knit', 'DeltaAnnotatedToFullText')
58
'breezy.knit', 'DeltaAnnotatedToFullText')
63
59
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
64
'bzrlib.knit', 'FTAnnotatedToUnannotated')
60
'breezy.knit', 'FTAnnotatedToUnannotated')
65
61
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
'bzrlib.knit', 'FTAnnotatedToFullText')
62
'breezy.knit', 'FTAnnotatedToFullText')
67
63
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
64
# 'breezy.knit', 'FTAnnotatedToChunked')
71
67
class ContentFactory(object):
205
class _MPDiffGenerator(object):
206
"""Pull out the functionality for generating mp_diffs."""
208
def __init__(self, vf, keys):
210
# This is the order the keys were requested in
211
self.ordered_keys = tuple(keys)
212
# keys + their parents, what we need to compute the diffs
213
self.needed_keys = ()
214
# Map from key: mp_diff
216
# Map from key: parents_needed (may have ghosts)
218
# Parents that aren't present
219
self.ghost_parents = ()
220
# Map from parent_key => number of children for this text
222
# Content chunks that are cached while we still need them
225
def _find_needed_keys(self):
226
"""Find the set of keys we need to request.
228
This includes all the original keys passed in, and the non-ghost
229
parents of those keys.
231
:return: (needed_keys, refcounts)
232
needed_keys is the set of all texts we need to extract
233
refcounts is a dict of {key: num_children} letting us know when we
234
no longer need to cache a given parent text
236
# All the keys and their parents
237
needed_keys = set(self.ordered_keys)
238
parent_map = self.vf.get_parent_map(needed_keys)
239
self.parent_map = parent_map
240
# TODO: Should we be using a different construct here? I think this
241
# uses difference_update internally, and we expect the result to
243
missing_keys = needed_keys.difference(parent_map)
245
raise errors.RevisionNotPresent(list(missing_keys)[0], self.vf)
246
# Parents that might be missing. They are allowed to be ghosts, but we
247
# should check for them
249
setdefault = refcounts.setdefault
251
for child_key, parent_keys in parent_map.iteritems():
253
# parent_keys may be None if a given VersionedFile claims to
254
# not support graph operations.
256
just_parents.update(parent_keys)
257
needed_keys.update(parent_keys)
258
for p in parent_keys:
259
refcounts[p] = setdefault(p, 0) + 1
260
just_parents.difference_update(parent_map)
261
# Remove any parents that are actually ghosts from the needed set
262
self.present_parents = set(self.vf.get_parent_map(just_parents))
263
self.ghost_parents = just_parents.difference(self.present_parents)
264
needed_keys.difference_update(self.ghost_parents)
265
self.needed_keys = needed_keys
266
self.refcounts = refcounts
267
return needed_keys, refcounts
269
def _compute_diff(self, key, parent_lines, lines):
270
"""Compute a single mp_diff, and store it in self._diffs"""
271
if len(parent_lines) > 0:
272
# XXX: _extract_blocks is not usefully defined anywhere...
273
# It was meant to extract the left-parent diff without
274
# having to recompute it for Knit content (pack-0.92,
275
# etc). That seems to have regressed somewhere
276
left_parent_blocks = self.vf._extract_blocks(key,
277
parent_lines[0], lines)
279
left_parent_blocks = None
280
diff = multiparent.MultiParent.from_lines(lines,
281
parent_lines, left_parent_blocks)
282
self.diffs[key] = diff
284
def _process_one_record(self, key, this_chunks):
286
if key in self.parent_map:
287
# This record should be ready to diff, since we requested
288
# content in 'topological' order
289
parent_keys = self.parent_map.pop(key)
290
# If a VersionedFile claims 'no-graph' support, then it may return
291
# None for any parent request, so we replace it with an empty tuple
292
if parent_keys is None:
295
for p in parent_keys:
296
# Alternatively we could check p not in self.needed_keys, but
297
# ghost_parents should be tiny versus huge
298
if p in self.ghost_parents:
300
refcount = self.refcounts[p]
301
if refcount == 1: # Last child reference
302
self.refcounts.pop(p)
303
parent_chunks = self.chunks.pop(p)
305
self.refcounts[p] = refcount - 1
306
parent_chunks = self.chunks[p]
307
p_lines = osutils.chunks_to_lines(parent_chunks)
308
# TODO: Should we cache the line form? We did the
309
# computation to get it, but storing it this way will
310
# be less memory efficient...
311
parent_lines.append(p_lines)
313
lines = osutils.chunks_to_lines(this_chunks)
314
# Since we needed the lines, we'll go ahead and cache them this way
316
self._compute_diff(key, parent_lines, lines)
318
# Is this content required for any more children?
319
if key in self.refcounts:
320
self.chunks[key] = this_chunks
322
def _extract_diffs(self):
323
needed_keys, refcounts = self._find_needed_keys()
324
for record in self.vf.get_record_stream(needed_keys,
325
'topological', True):
326
if record.storage_kind == 'absent':
327
raise errors.RevisionNotPresent(record.key, self.vf)
328
self._process_one_record(record.key,
329
record.get_bytes_as('chunked'))
331
def compute_diffs(self):
332
self._extract_diffs()
333
dpop = self.diffs.pop
334
return [dpop(k) for k in self.ordered_keys]
209
337
class VersionedFile(object):
210
338
"""Versioned text file storage.
790
922
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
791
923
may have a different length key-size, but that size will be constant for
792
all texts added to or retrieved from it. For instance, bzrlib uses
924
all texts added to or retrieved from it. For instance, breezy uses
793
925
instances with a key-size of 2 for storing user files in a repository, with
794
926
the first element the fileid, and the second the version of that file.
796
928
The use of tuples allows a single code base to support several different
797
929
uses with only the mapping logic changing from instance to instance.
931
:ivar _immediate_fallback_vfs: For subclasses that support stacking,
932
this is a list of other VersionedFiles immediately underneath this
933
one. They may in turn each have further fallbacks.
800
936
def add_lines(self, key, parents, lines, parent_texts=None,
1048
1184
def make_mpdiffs(self, keys):
1049
1185
"""Create multiparent diffs for specified keys."""
1050
keys_order = tuple(keys)
1051
keys = frozenset(keys)
1052
knit_keys = set(keys)
1053
parent_map = self.get_parent_map(keys)
1054
for parent_keys in parent_map.itervalues():
1056
knit_keys.update(parent_keys)
1057
missing_keys = keys - set(parent_map)
1059
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1060
# We need to filter out ghosts, because we can't diff against them.
1061
maybe_ghosts = knit_keys - keys
1062
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1063
knit_keys.difference_update(ghosts)
1065
chunks_to_lines = osutils.chunks_to_lines
1066
for record in self.get_record_stream(knit_keys, 'topological', True):
1067
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1068
# line_block_dict = {}
1069
# for parent, blocks in record.extract_line_blocks():
1070
# line_blocks[parent] = blocks
1071
# line_blocks[record.key] = line_block_dict
1073
for key in keys_order:
1075
parents = parent_map[key] or []
1076
# Note that filtering knit_keys can lead to a parent difference
1077
# between the creation and the application of the mpdiff.
1078
parent_lines = [lines[p] for p in parents if p in knit_keys]
1079
if len(parent_lines) > 0:
1080
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1083
left_parent_blocks = None
1084
diffs.append(multiparent.MultiParent.from_lines(target,
1085
parent_lines, left_parent_blocks))
1186
generator = _MPDiffGenerator(self, keys)
1187
return generator.compute_diffs()
1189
def get_annotator(self):
1190
return annotate.Annotator(self)
1088
1192
missing_keys = index._missing_keys_from_parent_map
1090
1194
def _extract_blocks(self, version_id, source, target):
1197
def _transitive_fallbacks(self):
1198
"""Return the whole stack of fallback versionedfiles.
1200
This VersionedFiles may have a list of fallbacks, but it doesn't
1201
necessarily know about the whole stack going down, and it can't know
1202
at open time because they may change after the objects are opened.
1205
for a_vfs in self._immediate_fallback_vfs:
1206
all_fallbacks.append(a_vfs)
1207
all_fallbacks.extend(a_vfs._transitive_fallbacks())
1208
return all_fallbacks
1094
1211
class ThunkedVersionedFiles(VersionedFiles):
1095
1212
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1428
class VersionedFilesWithFallbacks(VersionedFiles):
1430
def without_fallbacks(self):
1431
"""Return a clone of this object without any fallbacks configured."""
1432
raise NotImplementedError(self.without_fallbacks)
1434
def add_fallback_versioned_files(self, a_versioned_files):
1435
"""Add a source of texts for texts not present in this knit.
1437
:param a_versioned_files: A VersionedFiles object.
1439
raise NotImplementedError(self.add_fallback_versioned_files)
1441
def get_known_graph_ancestry(self, keys):
1442
"""Get a KnownGraph instance with the ancestry of keys."""
1443
parent_map, missing_keys = self._index.find_ancestry(keys)
1444
for fallback in self._transitive_fallbacks():
1445
if not missing_keys:
1447
(f_parent_map, f_missing_keys) = fallback._index.find_ancestry(
1449
parent_map.update(f_parent_map)
1450
missing_keys = f_missing_keys
1451
kg = _mod_graph.KnownGraph(parent_map)
1314
1455
class _PlanMergeVersionedFile(VersionedFiles):
1315
1456
"""A VersionedFile for uncommitted and committed texts.
1769
class NoDupeAddLinesDecorator(object):
1770
"""Decorator for a VersionedFiles that skips doing an add_lines if the key
1774
def __init__(self, store):
1777
def add_lines(self, key, parents, lines, parent_texts=None,
1778
left_matching_blocks=None, nostore_sha=None, random_id=False,
1779
check_content=True):
1780
"""See VersionedFiles.add_lines.
1782
This implementation may return None as the third element of the return
1783
value when the original store wouldn't.
1786
raise NotImplementedError(
1787
"NoDupeAddLinesDecorator.add_lines does not implement the "
1788
"nostore_sha behaviour.")
1790
sha1 = osutils.sha_strings(lines)
1791
key = ("sha1:" + sha1,)
1794
if key in self._store.get_parent_map([key]):
1795
# This key has already been inserted, so don't do it again.
1797
sha1 = osutils.sha_strings(lines)
1798
return sha1, sum(map(len, lines)), None
1799
return self._store.add_lines(key, parents, lines,
1800
parent_texts=parent_texts,
1801
left_matching_blocks=left_matching_blocks,
1802
nostore_sha=nostore_sha, random_id=random_id,
1803
check_content=check_content)
1805
def __getattr__(self, name):
1806
return getattr(self._store, name)
1627
1809
def network_bytes_to_kind_and_offset(network_bytes):
1628
1810
"""Strip of a record kind from the front of network_bytes.
1720
1902
for prefix in sorted(per_prefix_map):
1721
1903
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
1722
1904
return present_keys
1907
class _KeyRefs(object):
1909
def __init__(self, track_new_keys=False):
1910
# dict mapping 'key' to 'set of keys referring to that key'
1913
# set remembering all new keys
1914
self.new_keys = set()
1916
self.new_keys = None
1922
self.new_keys.clear()
1924
def add_references(self, key, refs):
1925
# Record the new references
1926
for referenced in refs:
1928
needed_by = self.refs[referenced]
1930
needed_by = self.refs[referenced] = set()
1932
# Discard references satisfied by the new key
1935
def get_new_keys(self):
1936
return self.new_keys
1938
def get_unsatisfied_refs(self):
1939
return self.refs.iterkeys()
1941
def _satisfy_refs_for_key(self, key):
1945
# No keys depended on this key. That's ok.
1948
def add_key(self, key):
1949
# satisfy refs for key, and remember that we've seen this key.
1950
self._satisfy_refs_for_key(key)
1951
if self.new_keys is not None:
1952
self.new_keys.add(key)
1954
def satisfy_refs_for_keys(self, keys):
1956
self._satisfy_refs_for_key(key)
1958
def get_referrers(self):
1960
for referrers in self.refs.itervalues():
1961
result.update(referrers)
added
removed
breezy/versionedfile.py
