1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20
"""Versioned text file storage api."""
23
from cStringIO import StringIO
26
from zlib import adler32
28
from bzrlib.lazy_import import lazy_import
29
lazy_import(globals(), """
44
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
45
from bzrlib.transport.memory import MemoryTransport
47
from bzrlib.inter import InterObject
48
from bzrlib.registry import Registry
49
from bzrlib.symbol_versioning import *
50
from bzrlib.textmerge import TextMerge
51
from bzrlib import bencode
54
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
56
'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
59
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
'bzrlib.knit', 'DeltaAnnotatedToFullText')
63
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
64
'bzrlib.knit', 'FTAnnotatedToUnannotated')
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
'bzrlib.knit', 'FTAnnotatedToFullText')
67
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
71
class ContentFactory(object):
72
"""Abstract interface for insertion and retrieval from a VersionedFile.
74
:ivar sha1: None, or the sha1 of the content fulltext.
75
:ivar storage_kind: The native storage kind of this factory. One of
76
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
77
'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
78
'knit-annotated-delta-gz', 'knit-ft-gz', 'knit-delta-gz'.
79
:ivar key: The key of this content. Each key is a tuple with a single
81
:ivar parents: A tuple of parent keys for self.key. If the object has
82
no parent information, None (as opposed to () for an empty list of
87
"""Create a ContentFactory."""
89
self.storage_kind = None
94
class ChunkedContentFactory(ContentFactory):
95
"""Static data content factory.
97
This takes a 'chunked' list of strings. The only requirement on 'chunked' is
98
that ''.join(lines) becomes a valid fulltext. A tuple of a single string
99
satisfies this, as does a list of lines.
101
:ivar sha1: None, or the sha1 of the content fulltext.
102
:ivar storage_kind: The native storage kind of this factory. Always
104
:ivar key: The key of this content. Each key is a tuple with a single
106
:ivar parents: A tuple of parent keys for self.key. If the object has
107
no parent information, None (as opposed to () for an empty list of
111
def __init__(self, key, parents, sha1, chunks):
112
"""Create a ContentFactory."""
114
self.storage_kind = 'chunked'
116
self.parents = parents
117
self._chunks = chunks
119
def get_bytes_as(self, storage_kind):
120
if storage_kind == 'chunked':
122
elif storage_kind == 'fulltext':
123
return ''.join(self._chunks)
124
raise errors.UnavailableRepresentation(self.key, storage_kind,
128
class FulltextContentFactory(ContentFactory):
129
"""Static data content factory.
131
This takes a fulltext when created and just returns that during
132
get_bytes_as('fulltext').
134
:ivar sha1: None, or the sha1 of the content fulltext.
135
:ivar storage_kind: The native storage kind of this factory. Always
137
:ivar key: The key of this content. Each key is a tuple with a single
139
:ivar parents: A tuple of parent keys for self.key. If the object has
140
no parent information, None (as opposed to () for an empty list of
144
def __init__(self, key, parents, sha1, text):
145
"""Create a ContentFactory."""
147
self.storage_kind = 'fulltext'
149
self.parents = parents
152
def get_bytes_as(self, storage_kind):
153
if storage_kind == self.storage_kind:
155
elif storage_kind == 'chunked':
157
raise errors.UnavailableRepresentation(self.key, storage_kind,
161
class AbsentContentFactory(ContentFactory):
162
"""A placeholder content factory for unavailable texts.
165
:ivar storage_kind: 'absent'.
166
:ivar key: The key of this content. Each key is a tuple with a single
171
def __init__(self, key):
172
"""Create a ContentFactory."""
174
self.storage_kind = 'absent'
178
def get_bytes_as(self, storage_kind):
179
raise ValueError('A request was made for key: %s, but that'
180
' content is not available, and the calling'
181
' code does not handle if it is missing.'
185
class AdapterFactory(ContentFactory):
186
"""A content factory to adapt between key prefix's."""
188
def __init__(self, key, parents, adapted):
189
"""Create an adapter factory instance."""
191
self.parents = parents
192
self._adapted = adapted
194
def __getattr__(self, attr):
195
"""Return a member from the adapted object."""
196
if attr in ('key', 'parents'):
197
return self.__dict__[attr]
199
return getattr(self._adapted, attr)
202
def filter_absent(record_stream):
203
"""Adapt a record stream to remove absent records."""
204
for record in record_stream:
205
if record.storage_kind != 'absent':
209
class VersionedFile(object):
210
"""Versioned text file storage.
212
A versioned file manages versions of line-based text files,
213
keeping track of the originating version for each line.
215
To clients the "lines" of the file are represented as a list of
216
strings. These strings will typically have terminal newline
217
characters, but this is not required. In particular files commonly
218
do not have a newline at the end of the file.
220
Texts are identified by a version-id string.
224
def check_not_reserved_id(version_id):
225
revision.check_not_reserved_id(version_id)
227
def copy_to(self, name, transport):
228
"""Copy this versioned file to name on transport."""
229
raise NotImplementedError(self.copy_to)
231
def get_known_graph_ancestry(self, keys):
232
"""Get a KnownGraph instance with the ancestry of keys."""
233
raise NotImplementedError(self.get_known_graph_ancestry)
235
def get_record_stream(self, versions, ordering, include_delta_closure):
236
"""Get a stream of records for versions.
238
:param versions: The versions to include. Each version is a tuple
240
:param ordering: Either 'unordered' or 'topological'. A topologically
241
sorted stream has compression parents strictly before their
243
:param include_delta_closure: If True then the closure across any
244
compression parents will be included (in the data content of the
245
stream, not in the emitted records). This guarantees that
246
'fulltext' can be used successfully on every record.
247
:return: An iterator of ContentFactory objects, each of which is only
248
valid until the iterator is advanced.
250
raise NotImplementedError(self.get_record_stream)
252
def has_version(self, version_id):
253
"""Returns whether version is present."""
254
raise NotImplementedError(self.has_version)
256
def insert_record_stream(self, stream):
257
"""Insert a record stream into this versioned file.
259
:param stream: A stream of records to insert.
261
:seealso VersionedFile.get_record_stream:
263
raise NotImplementedError
265
def add_lines(self, version_id, parents, lines, parent_texts=None,
266
left_matching_blocks=None, nostore_sha=None, random_id=False,
268
"""Add a single text on top of the versioned file.
270
Must raise RevisionAlreadyPresent if the new version is
271
already present in file history.
273
Must raise RevisionNotPresent if any of the given parents are
274
not present in file history.
276
:param lines: A list of lines. Each line must be a bytestring. And all
277
of them except the last must be terminated with \n and contain no
278
other \n's. The last line may either contain no \n's or a single
279
terminated \n. If the lines list does meet this constraint the add
280
routine may error or may succeed - but you will be unable to read
281
the data back accurately. (Checking the lines have been split
282
correctly is expensive and extremely unlikely to catch bugs so it
283
is not done at runtime unless check_content is True.)
284
:param parent_texts: An optional dictionary containing the opaque
285
representations of some or all of the parents of version_id to
286
allow delta optimisations. VERY IMPORTANT: the texts must be those
287
returned by add_lines or data corruption can be caused.
288
:param left_matching_blocks: a hint about which areas are common
289
between the text and its left-hand-parent. The format is
290
the SequenceMatcher.get_matching_blocks format.
291
:param nostore_sha: Raise ExistingContent and do not add the lines to
292
the versioned file if the digest of the lines matches this.
293
:param random_id: If True a random id has been selected rather than
294
an id determined by some deterministic process such as a converter
295
from a foreign VCS. When True the backend may choose not to check
296
for uniqueness of the resulting key within the versioned file, so
297
this should only be done when the result is expected to be unique
299
:param check_content: If True, the lines supplied are verified to be
300
bytestrings that are correctly formed lines.
301
:return: The text sha1, the number of bytes in the text, and an opaque
302
representation of the inserted version which can be provided
303
back to future add_lines calls in the parent_texts dictionary.
305
self._check_write_ok()
306
return self._add_lines(version_id, parents, lines, parent_texts,
307
left_matching_blocks, nostore_sha, random_id, check_content)
309
def _add_lines(self, version_id, parents, lines, parent_texts,
310
left_matching_blocks, nostore_sha, random_id, check_content):
311
"""Helper to do the class specific add_lines."""
312
raise NotImplementedError(self.add_lines)
314
def add_lines_with_ghosts(self, version_id, parents, lines,
315
parent_texts=None, nostore_sha=None, random_id=False,
316
check_content=True, left_matching_blocks=None):
317
"""Add lines to the versioned file, allowing ghosts to be present.
319
This takes the same parameters as add_lines and returns the same.
321
self._check_write_ok()
322
return self._add_lines_with_ghosts(version_id, parents, lines,
323
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
325
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
326
nostore_sha, random_id, check_content, left_matching_blocks):
327
"""Helper to do class specific add_lines_with_ghosts."""
328
raise NotImplementedError(self.add_lines_with_ghosts)
330
def check(self, progress_bar=None):
331
"""Check the versioned file for integrity."""
332
raise NotImplementedError(self.check)
334
def _check_lines_not_unicode(self, lines):
335
"""Check that lines being added to a versioned file are not unicode."""
337
if line.__class__ is not str:
338
raise errors.BzrBadParameterUnicode("lines")
340
def _check_lines_are_lines(self, lines):
341
"""Check that the lines really are full lines without inline EOL."""
343
if '\n' in line[:-1]:
344
raise errors.BzrBadParameterContainsNewline("lines")
346
def get_format_signature(self):
347
"""Get a text description of the data encoding in this file.
351
raise NotImplementedError(self.get_format_signature)
353
def make_mpdiffs(self, version_ids):
354
"""Create multiparent diffs for specified versions."""
355
knit_versions = set()
356
knit_versions.update(version_ids)
357
parent_map = self.get_parent_map(version_ids)
358
for version_id in version_ids:
360
knit_versions.update(parent_map[version_id])
362
raise errors.RevisionNotPresent(version_id, self)
363
# We need to filter out ghosts, because we can't diff against them.
364
knit_versions = set(self.get_parent_map(knit_versions).keys())
365
lines = dict(zip(knit_versions,
366
self._get_lf_split_line_list(knit_versions)))
368
for version_id in version_ids:
369
target = lines[version_id]
371
parents = [lines[p] for p in parent_map[version_id] if p in
374
# I don't know how this could ever trigger.
375
# parent_map[version_id] was already triggered in the previous
376
# for loop, and lines[p] has the 'if p in knit_versions' check,
377
# so we again won't have a KeyError.
378
raise errors.RevisionNotPresent(version_id, self)
380
left_parent_blocks = self._extract_blocks(version_id,
383
left_parent_blocks = None
384
diffs.append(multiparent.MultiParent.from_lines(target, parents,
388
def _extract_blocks(self, version_id, source, target):
391
def add_mpdiffs(self, records):
392
"""Add mpdiffs to this VersionedFile.
394
Records should be iterables of version, parents, expected_sha1,
395
mpdiff. mpdiff should be a MultiParent instance.
397
# Does this need to call self._check_write_ok()? (IanC 20070919)
399
mpvf = multiparent.MultiMemoryVersionedFile()
401
for version, parent_ids, expected_sha1, mpdiff in records:
402
versions.append(version)
403
mpvf.add_diff(mpdiff, version, parent_ids)
404
needed_parents = set()
405
for version, parent_ids, expected_sha1, mpdiff in records:
406
needed_parents.update(p for p in parent_ids
407
if not mpvf.has_version(p))
408
present_parents = set(self.get_parent_map(needed_parents).keys())
409
for parent_id, lines in zip(present_parents,
410
self._get_lf_split_line_list(present_parents)):
411
mpvf.add_version(lines, parent_id, [])
412
for (version, parent_ids, expected_sha1, mpdiff), lines in\
413
zip(records, mpvf.get_line_list(versions)):
414
if len(parent_ids) == 1:
415
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
416
mpvf.get_diff(parent_ids[0]).num_lines()))
418
left_matching_blocks = None
420
_, _, version_text = self.add_lines_with_ghosts(version,
421
parent_ids, lines, vf_parents,
422
left_matching_blocks=left_matching_blocks)
423
except NotImplementedError:
424
# The vf can't handle ghosts, so add lines normally, which will
425
# (reasonably) fail if there are ghosts in the data.
426
_, _, version_text = self.add_lines(version,
427
parent_ids, lines, vf_parents,
428
left_matching_blocks=left_matching_blocks)
429
vf_parents[version] = version_text
430
sha1s = self.get_sha1s(versions)
431
for version, parent_ids, expected_sha1, mpdiff in records:
432
if expected_sha1 != sha1s[version]:
433
raise errors.VersionedFileInvalidChecksum(version)
435
def get_text(self, version_id):
436
"""Return version contents as a text string.
438
Raises RevisionNotPresent if version is not present in
441
return ''.join(self.get_lines(version_id))
442
get_string = get_text
444
def get_texts(self, version_ids):
445
"""Return the texts of listed versions as a list of strings.
447
Raises RevisionNotPresent if version is not present in
450
return [''.join(self.get_lines(v)) for v in version_ids]
452
def get_lines(self, version_id):
453
"""Return version contents as a sequence of lines.
455
Raises RevisionNotPresent if version is not present in
458
raise NotImplementedError(self.get_lines)
460
def _get_lf_split_line_list(self, version_ids):
461
return [StringIO(t).readlines() for t in self.get_texts(version_ids)]
463
def get_ancestry(self, version_ids, topo_sorted=True):
464
"""Return a list of all ancestors of given version(s). This
465
will not include the null revision.
467
This list will not be topologically sorted if topo_sorted=False is
470
Must raise RevisionNotPresent if any of the given versions are
471
not present in file history."""
472
if isinstance(version_ids, basestring):
473
version_ids = [version_ids]
474
raise NotImplementedError(self.get_ancestry)
476
def get_ancestry_with_ghosts(self, version_ids):
477
"""Return a list of all ancestors of given version(s). This
478
will not include the null revision.
480
Must raise RevisionNotPresent if any of the given versions are
481
not present in file history.
483
Ghosts that are known about will be included in ancestry list,
484
but are not explicitly marked.
486
raise NotImplementedError(self.get_ancestry_with_ghosts)
488
def get_parent_map(self, version_ids):
489
"""Get a map of the parents of version_ids.
491
:param version_ids: The version ids to look up parents for.
492
:return: A mapping from version id to parents.
494
raise NotImplementedError(self.get_parent_map)
496
def get_parents_with_ghosts(self, version_id):
497
"""Return version names for parents of version_id.
499
Will raise RevisionNotPresent if version_id is not present
502
Ghosts that are known about will be included in the parent list,
503
but are not explicitly marked.
506
return list(self.get_parent_map([version_id])[version_id])
508
raise errors.RevisionNotPresent(version_id, self)
510
def annotate(self, version_id):
511
"""Return a list of (version-id, line) tuples for version_id.
513
:raise RevisionNotPresent: If the given version is
514
not present in file history.
516
raise NotImplementedError(self.annotate)
518
def iter_lines_added_or_present_in_versions(self, version_ids=None,
520
"""Iterate over the lines in the versioned file from version_ids.
522
This may return lines from other versions. Each item the returned
523
iterator yields is a tuple of a line and a text version that that line
524
is present in (not introduced in).
526
Ordering of results is in whatever order is most suitable for the
527
underlying storage format.
529
If a progress bar is supplied, it may be used to indicate progress.
530
The caller is responsible for cleaning up progress bars (because this
533
NOTES: Lines are normalised: they will all have \n terminators.
534
Lines are returned in arbitrary order.
536
:return: An iterator over (line, version_id).
538
raise NotImplementedError(self.iter_lines_added_or_present_in_versions)
540
def plan_merge(self, ver_a, ver_b):
541
"""Return pseudo-annotation indicating how the two versions merge.
543
This is computed between versions a and b and their common
546
Weave lines present in none of them are skipped entirely.
549
killed-base Dead in base revision
550
killed-both Killed in each revision
553
unchanged Alive in both a and b (possibly created in both)
556
ghost-a Killed in a, unborn in b
557
ghost-b Killed in b, unborn in a
558
irrelevant Not in either revision
560
raise NotImplementedError(VersionedFile.plan_merge)
562
def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
563
b_marker=TextMerge.B_MARKER):
564
return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
567
class RecordingVersionedFilesDecorator(object):
568
"""A minimal versioned files that records calls made on it.
570
Only enough methods have been added to support tests using it to date.
572
:ivar calls: A list of the calls made; can be reset at any time by
576
def __init__(self, backing_vf):
577
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
579
:param backing_vf: The versioned file to answer all methods.
581
self._backing_vf = backing_vf
584
def add_lines(self, key, parents, lines, parent_texts=None,
585
left_matching_blocks=None, nostore_sha=None, random_id=False,
587
self.calls.append(("add_lines", key, parents, lines, parent_texts,
588
left_matching_blocks, nostore_sha, random_id, check_content))
589
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
590
left_matching_blocks, nostore_sha, random_id, check_content)
593
self._backing_vf.check()
595
def get_parent_map(self, keys):
596
self.calls.append(("get_parent_map", copy(keys)))
597
return self._backing_vf.get_parent_map(keys)
599
def get_record_stream(self, keys, sort_order, include_delta_closure):
600
self.calls.append(("get_record_stream", list(keys), sort_order,
601
include_delta_closure))
602
return self._backing_vf.get_record_stream(keys, sort_order,
603
include_delta_closure)
605
def get_sha1s(self, keys):
606
self.calls.append(("get_sha1s", copy(keys)))
607
return self._backing_vf.get_sha1s(keys)
609
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
610
self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
611
return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
614
self.calls.append(("keys",))
615
return self._backing_vf.keys()
618
class OrderingVersionedFilesDecorator(RecordingVersionedFilesDecorator):
619
"""A VF that records calls, and returns keys in specific order.
621
:ivar calls: A list of the calls made; can be reset at any time by
625
def __init__(self, backing_vf, key_priority):
626
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
628
:param backing_vf: The versioned file to answer all methods.
629
:param key_priority: A dictionary defining what order keys should be
630
returned from an 'unordered' get_record_stream request.
631
Keys with lower priority are returned first, keys not present in
632
the map get an implicit priority of 0, and are returned in
633
lexicographical order.
635
RecordingVersionedFilesDecorator.__init__(self, backing_vf)
636
self._key_priority = key_priority
638
def get_record_stream(self, keys, sort_order, include_delta_closure):
639
self.calls.append(("get_record_stream", list(keys), sort_order,
640
include_delta_closure))
641
if sort_order == 'unordered':
643
return (self._key_priority.get(key, 0), key)
644
# Use a defined order by asking for the keys one-by-one from the
646
for key in sorted(keys, key=sort_key):
647
for record in self._backing_vf.get_record_stream([key],
648
'unordered', include_delta_closure):
651
for record in self._backing_vf.get_record_stream(keys, sort_order,
652
include_delta_closure):
656
class KeyMapper(object):
657
"""KeyMappers map between keys and underlying partitioned storage."""
660
"""Map key to an underlying storage identifier.
662
:param key: A key tuple e.g. ('file-id', 'revision-id').
663
:return: An underlying storage identifier, specific to the partitioning
666
raise NotImplementedError(self.map)
668
def unmap(self, partition_id):
669
"""Map a partitioned storage id back to a key prefix.
671
:param partition_id: The underlying partition id.
672
:return: As much of a key (or prefix) as is derivable from the partition
675
raise NotImplementedError(self.unmap)
678
class ConstantMapper(KeyMapper):
679
"""A key mapper that maps to a constant result."""
681
def __init__(self, result):
682
"""Create a ConstantMapper which will return result for all maps."""
683
self._result = result
686
"""See KeyMapper.map()."""
690
class URLEscapeMapper(KeyMapper):
691
"""Base class for use with transport backed storage.
693
This provides a map and unmap wrapper that respectively url escape and
694
unescape their outputs and inputs.
698
"""See KeyMapper.map()."""
699
return urllib.quote(self._map(key))
701
def unmap(self, partition_id):
702
"""See KeyMapper.unmap()."""
703
return self._unmap(urllib.unquote(partition_id))
706
class PrefixMapper(URLEscapeMapper):
707
"""A key mapper that extracts the first component of a key.
709
This mapper is for use with a transport based backend.
713
"""See KeyMapper.map()."""
716
def _unmap(self, partition_id):
717
"""See KeyMapper.unmap()."""
718
return (partition_id,)
721
class HashPrefixMapper(URLEscapeMapper):
722
"""A key mapper that combines the first component of a key with a hash.
724
This mapper is for use with a transport based backend.
728
"""See KeyMapper.map()."""
729
prefix = self._escape(key[0])
730
return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
732
def _escape(self, prefix):
733
"""No escaping needed here."""
736
def _unmap(self, partition_id):
737
"""See KeyMapper.unmap()."""
738
return (self._unescape(osutils.basename(partition_id)),)
740
def _unescape(self, basename):
741
"""No unescaping needed for HashPrefixMapper."""
745
class HashEscapedPrefixMapper(HashPrefixMapper):
746
"""Combines the escaped first component of a key with a hash.
748
This mapper is for use with a transport based backend.
751
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
753
def _escape(self, prefix):
754
"""Turn a key element into a filesystem safe string.
756
This is similar to a plain urllib.quote, except
757
it uses specific safe characters, so that it doesn't
758
have to translate a lot of valid file ids.
760
# @ does not get escaped. This is because it is a valid
761
# filesystem character we use all the time, and it looks
762
# a lot better than seeing %40 all the time.
763
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
767
def _unescape(self, basename):
768
"""Escaped names are easily unescaped by urlutils."""
769
return urllib.unquote(basename)
772
def make_versioned_files_factory(versioned_file_factory, mapper):
773
"""Create a ThunkedVersionedFiles factory.
775
This will create a callable which when called creates a
776
ThunkedVersionedFiles on a transport, using mapper to access individual
777
versioned files, and versioned_file_factory to create each individual file.
779
def factory(transport):
780
return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
785
class VersionedFiles(object):
786
"""Storage for many versioned files.
788
This object allows a single keyspace for accessing the history graph and
789
contents of named bytestrings.
791
Currently no implementation allows the graph of different key prefixes to
792
intersect, but the API does allow such implementations in the future.
794
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
795
may have a different length key-size, but that size will be constant for
796
all texts added to or retrieved from it. For instance, bzrlib uses
797
instances with a key-size of 2 for storing user files in a repository, with
798
the first element the fileid, and the second the version of that file.
800
The use of tuples allows a single code base to support several different
801
uses with only the mapping logic changing from instance to instance.
804
def add_lines(self, key, parents, lines, parent_texts=None,
805
left_matching_blocks=None, nostore_sha=None, random_id=False,
807
"""Add a text to the store.
809
:param key: The key tuple of the text to add. If the last element is
810
None, a CHK string will be generated during the addition.
811
:param parents: The parents key tuples of the text to add.
812
:param lines: A list of lines. Each line must be a bytestring. And all
813
of them except the last must be terminated with \n and contain no
814
other \n's. The last line may either contain no \n's or a single
815
terminating \n. If the lines list does meet this constraint the add
816
routine may error or may succeed - but you will be unable to read
817
the data back accurately. (Checking the lines have been split
818
correctly is expensive and extremely unlikely to catch bugs so it
819
is not done at runtime unless check_content is True.)
820
:param parent_texts: An optional dictionary containing the opaque
821
representations of some or all of the parents of version_id to
822
allow delta optimisations. VERY IMPORTANT: the texts must be those
823
returned by add_lines or data corruption can be caused.
824
:param left_matching_blocks: a hint about which areas are common
825
between the text and its left-hand-parent. The format is
826
the SequenceMatcher.get_matching_blocks format.
827
:param nostore_sha: Raise ExistingContent and do not add the lines to
828
the versioned file if the digest of the lines matches this.
829
:param random_id: If True a random id has been selected rather than
830
an id determined by some deterministic process such as a converter
831
from a foreign VCS. When True the backend may choose not to check
832
for uniqueness of the resulting key within the versioned file, so
833
this should only be done when the result is expected to be unique
835
:param check_content: If True, the lines supplied are verified to be
836
bytestrings that are correctly formed lines.
837
:return: The text sha1, the number of bytes in the text, and an opaque
838
representation of the inserted version which can be provided
839
back to future add_lines calls in the parent_texts dictionary.
841
raise NotImplementedError(self.add_lines)
843
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
844
"""Add a text to the store.
846
This is a private function for use by CommitBuilder.
848
:param key: The key tuple of the text to add. If the last element is
849
None, a CHK string will be generated during the addition.
850
:param parents: The parents key tuples of the text to add.
851
:param text: A string containing the text to be committed.
852
:param nostore_sha: Raise ExistingContent and do not add the lines to
853
the versioned file if the digest of the lines matches this.
854
:param random_id: If True a random id has been selected rather than
855
an id determined by some deterministic process such as a converter
856
from a foreign VCS. When True the backend may choose not to check
857
for uniqueness of the resulting key within the versioned file, so
858
this should only be done when the result is expected to be unique
860
:param check_content: If True, the lines supplied are verified to be
861
bytestrings that are correctly formed lines.
862
:return: The text sha1, the number of bytes in the text, and an opaque
863
representation of the inserted version which can be provided
864
back to future _add_text calls in the parent_texts dictionary.
866
# The default implementation just thunks over to .add_lines(),
867
# inefficient, but it works.
868
return self.add_lines(key, parents, osutils.split_lines(text),
869
nostore_sha=nostore_sha,
873
def add_mpdiffs(self, records):
874
"""Add mpdiffs to this VersionedFile.
876
Records should be iterables of version, parents, expected_sha1,
877
mpdiff. mpdiff should be a MultiParent instance.
880
mpvf = multiparent.MultiMemoryVersionedFile()
882
for version, parent_ids, expected_sha1, mpdiff in records:
883
versions.append(version)
884
mpvf.add_diff(mpdiff, version, parent_ids)
885
needed_parents = set()
886
for version, parent_ids, expected_sha1, mpdiff in records:
887
needed_parents.update(p for p in parent_ids
888
if not mpvf.has_version(p))
889
# It seems likely that adding all the present parents as fulltexts can
890
# easily exhaust memory.
891
chunks_to_lines = osutils.chunks_to_lines
892
for record in self.get_record_stream(needed_parents, 'unordered',
894
if record.storage_kind == 'absent':
896
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
898
for (key, parent_keys, expected_sha1, mpdiff), lines in\
899
zip(records, mpvf.get_line_list(versions)):
900
if len(parent_keys) == 1:
901
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
902
mpvf.get_diff(parent_keys[0]).num_lines()))
904
left_matching_blocks = None
905
version_sha1, _, version_text = self.add_lines(key,
906
parent_keys, lines, vf_parents,
907
left_matching_blocks=left_matching_blocks)
908
if version_sha1 != expected_sha1:
909
raise errors.VersionedFileInvalidChecksum(version)
910
vf_parents[key] = version_text
912
def annotate(self, key):
913
"""Return a list of (version-key, line) tuples for the text of key.
915
:raise RevisionNotPresent: If the key is not present.
917
raise NotImplementedError(self.annotate)
919
def check(self, progress_bar=None):
920
"""Check this object for integrity.
922
:param progress_bar: A progress bar to output as the check progresses.
923
:param keys: Specific keys within the VersionedFiles to check. When
924
this parameter is not None, check() becomes a generator as per
925
get_record_stream. The difference to get_record_stream is that
926
more or deeper checks will be performed.
927
:return: None, or if keys was supplied a generator as per
930
raise NotImplementedError(self.check)
933
def check_not_reserved_id(version_id):
934
revision.check_not_reserved_id(version_id)
936
def _check_lines_not_unicode(self, lines):
937
"""Check that lines being added to a versioned file are not unicode."""
939
if line.__class__ is not str:
940
raise errors.BzrBadParameterUnicode("lines")
942
def _check_lines_are_lines(self, lines):
943
"""Check that the lines really are full lines without inline EOL."""
945
if '\n' in line[:-1]:
946
raise errors.BzrBadParameterContainsNewline("lines")
948
def get_parent_map(self, keys):
949
"""Get a map of the parents of keys.
951
:param keys: The keys to look up parents for.
952
:return: A mapping from keys to parents. Absent keys are absent from
955
raise NotImplementedError(self.get_parent_map)
957
def get_record_stream(self, keys, ordering, include_delta_closure):
958
"""Get a stream of records for keys.
960
:param keys: The keys to include.
961
:param ordering: Either 'unordered' or 'topological'. A topologically
962
sorted stream has compression parents strictly before their
964
:param include_delta_closure: If True then the closure across any
965
compression parents will be included (in the opaque data).
966
:return: An iterator of ContentFactory objects, each of which is only
967
valid until the iterator is advanced.
969
raise NotImplementedError(self.get_record_stream)
971
def get_sha1s(self, keys):
972
"""Get the sha1's of the texts for the given keys.
974
:param keys: The names of the keys to lookup
975
:return: a dict from key to sha1 digest. Keys of texts which are not
976
present in the store are not present in the returned
979
raise NotImplementedError(self.get_sha1s)
981
has_key = index._has_key_from_parent_map
983
def get_missing_compression_parent_keys(self):
984
"""Return an iterable of keys of missing compression parents.
986
Check this after calling insert_record_stream to find out if there are
987
any missing compression parents. If there are, the records that
988
depend on them are not able to be inserted safely. The precise
989
behaviour depends on the concrete VersionedFiles class in use.
991
Classes that do not support this will raise NotImplementedError.
993
raise NotImplementedError(self.get_missing_compression_parent_keys)
995
def insert_record_stream(self, stream):
996
"""Insert a record stream into this container.
998
:param stream: A stream of records to insert.
1000
:seealso VersionedFile.get_record_stream:
1002
raise NotImplementedError
1004
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1005
"""Iterate over the lines in the versioned files from keys.
1007
This may return lines from other keys. Each item the returned
1008
iterator yields is a tuple of a line and a text version that that line
1009
is present in (not introduced in).
1011
Ordering of results is in whatever order is most suitable for the
1012
underlying storage format.
1014
If a progress bar is supplied, it may be used to indicate progress.
1015
The caller is responsible for cleaning up progress bars (because this
1019
* Lines are normalised by the underlying store: they will all have \n
1021
* Lines are returned in arbitrary order.
1023
:return: An iterator over (line, key).
1025
raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
1028
"""Return a iterable of the keys for all the contained texts."""
1029
raise NotImplementedError(self.keys)
1031
def make_mpdiffs(self, keys):
1032
"""Create multiparent diffs for specified keys."""
1033
keys_order = tuple(keys)
1034
keys = frozenset(keys)
1035
knit_keys = set(keys)
1036
parent_map = self.get_parent_map(keys)
1037
for parent_keys in parent_map.itervalues():
1039
knit_keys.update(parent_keys)
1040
missing_keys = keys - set(parent_map)
1042
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1043
# We need to filter out ghosts, because we can't diff against them.
1044
maybe_ghosts = knit_keys - keys
1045
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1046
knit_keys.difference_update(ghosts)
1048
chunks_to_lines = osutils.chunks_to_lines
1049
for record in self.get_record_stream(knit_keys, 'topological', True):
1050
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1051
# line_block_dict = {}
1052
# for parent, blocks in record.extract_line_blocks():
1053
# line_blocks[parent] = blocks
1054
# line_blocks[record.key] = line_block_dict
1056
for key in keys_order:
1058
parents = parent_map[key] or []
1059
# Note that filtering knit_keys can lead to a parent difference
1060
# between the creation and the application of the mpdiff.
1061
parent_lines = [lines[p] for p in parents if p in knit_keys]
1062
if len(parent_lines) > 0:
1063
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1066
left_parent_blocks = None
1067
diffs.append(multiparent.MultiParent.from_lines(target,
1068
parent_lines, left_parent_blocks))
1071
missing_keys = index._missing_keys_from_parent_map
1073
def _extract_blocks(self, version_id, source, target):
1077
class ThunkedVersionedFiles(VersionedFiles):
1078
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1080
This object allows a single keyspace for accessing the history graph and
1081
contents of named bytestrings.
1083
Currently no implementation allows the graph of different key prefixes to
1084
intersect, but the API does allow such implementations in the future.
1087
def __init__(self, transport, file_factory, mapper, is_locked):
1088
"""Create a ThunkedVersionedFiles."""
1089
self._transport = transport
1090
self._file_factory = file_factory
1091
self._mapper = mapper
1092
self._is_locked = is_locked
1094
def add_lines(self, key, parents, lines, parent_texts=None,
1095
left_matching_blocks=None, nostore_sha=None, random_id=False,
1096
check_content=True):
1097
"""See VersionedFiles.add_lines()."""
1098
path = self._mapper.map(key)
1099
version_id = key[-1]
1100
parents = [parent[-1] for parent in parents]
1101
vf = self._get_vf(path)
1104
return vf.add_lines_with_ghosts(version_id, parents, lines,
1105
parent_texts=parent_texts,
1106
left_matching_blocks=left_matching_blocks,
1107
nostore_sha=nostore_sha, random_id=random_id,
1108
check_content=check_content)
1109
except NotImplementedError:
1110
return vf.add_lines(version_id, parents, lines,
1111
parent_texts=parent_texts,
1112
left_matching_blocks=left_matching_blocks,
1113
nostore_sha=nostore_sha, random_id=random_id,
1114
check_content=check_content)
1115
except errors.NoSuchFile:
1116
# parent directory may be missing, try again.
1117
self._transport.mkdir(osutils.dirname(path))
1119
return vf.add_lines_with_ghosts(version_id, parents, lines,
1120
parent_texts=parent_texts,
1121
left_matching_blocks=left_matching_blocks,
1122
nostore_sha=nostore_sha, random_id=random_id,
1123
check_content=check_content)
1124
except NotImplementedError:
1125
return vf.add_lines(version_id, parents, lines,
1126
parent_texts=parent_texts,
1127
left_matching_blocks=left_matching_blocks,
1128
nostore_sha=nostore_sha, random_id=random_id,
1129
check_content=check_content)
1131
def annotate(self, key):
1132
"""Return a list of (version-key, line) tuples for the text of key.
1134
:raise RevisionNotPresent: If the key is not present.
1137
path = self._mapper.map(prefix)
1138
vf = self._get_vf(path)
1139
origins = vf.annotate(key[-1])
1141
for origin, line in origins:
1142
result.append((prefix + (origin,), line))
1145
def get_annotator(self):
1146
return annotate.Annotator(self)
1148
def check(self, progress_bar=None, keys=None):
1149
"""See VersionedFiles.check()."""
1150
# XXX: This is over-enthusiastic but as we only thunk for Weaves today
1151
# this is tolerable. Ideally we'd pass keys down to check() and
1152
# have the older VersiondFile interface updated too.
1153
for prefix, vf in self._iter_all_components():
1155
if keys is not None:
1156
return self.get_record_stream(keys, 'unordered', True)
1158
def get_parent_map(self, keys):
1159
"""Get a map of the parents of keys.
1161
:param keys: The keys to look up parents for.
1162
:return: A mapping from keys to parents. Absent keys are absent from
1165
prefixes = self._partition_keys(keys)
1167
for prefix, suffixes in prefixes.items():
1168
path = self._mapper.map(prefix)
1169
vf = self._get_vf(path)
1170
parent_map = vf.get_parent_map(suffixes)
1171
for key, parents in parent_map.items():
1172
result[prefix + (key,)] = tuple(
1173
prefix + (parent,) for parent in parents)
1176
def _get_vf(self, path):
1177
if not self._is_locked():
1178
raise errors.ObjectNotLocked(self)
1179
return self._file_factory(path, self._transport, create=True,
1180
get_scope=lambda:None)
1182
def _partition_keys(self, keys):
1183
"""Turn keys into a dict of prefix:suffix_list."""
1186
prefix_keys = result.setdefault(key[:-1], [])
1187
prefix_keys.append(key[-1])
1190
def _get_all_prefixes(self):
1191
# Identify all key prefixes.
1192
# XXX: A bit hacky, needs polish.
1193
if type(self._mapper) == ConstantMapper:
1194
paths = [self._mapper.map(())]
1198
for quoted_relpath in self._transport.iter_files_recursive():
1199
path, ext = os.path.splitext(quoted_relpath)
1201
paths = list(relpaths)
1202
prefixes = [self._mapper.unmap(path) for path in paths]
1203
return zip(paths, prefixes)
1205
def get_record_stream(self, keys, ordering, include_delta_closure):
1206
"""See VersionedFiles.get_record_stream()."""
1207
# Ordering will be taken care of by each partitioned store; group keys
1210
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1211
suffixes = [(suffix,) for suffix in suffixes]
1212
for record in vf.get_record_stream(suffixes, ordering,
1213
include_delta_closure):
1214
if record.parents is not None:
1215
record.parents = tuple(
1216
prefix + parent for parent in record.parents)
1217
record.key = prefix + record.key
1220
def _iter_keys_vf(self, keys):
1221
prefixes = self._partition_keys(keys)
1223
for prefix, suffixes in prefixes.items():
1224
path = self._mapper.map(prefix)
1225
vf = self._get_vf(path)
1226
yield prefix, suffixes, vf
1228
def get_sha1s(self, keys):
1229
"""See VersionedFiles.get_sha1s()."""
1231
for prefix,suffixes, vf in self._iter_keys_vf(keys):
1232
vf_sha1s = vf.get_sha1s(suffixes)
1233
for suffix, sha1 in vf_sha1s.iteritems():
1234
sha1s[prefix + (suffix,)] = sha1
1237
def insert_record_stream(self, stream):
1238
"""Insert a record stream into this container.
1240
:param stream: A stream of records to insert.
1242
:seealso VersionedFile.get_record_stream:
1244
for record in stream:
1245
prefix = record.key[:-1]
1246
key = record.key[-1:]
1247
if record.parents is not None:
1248
parents = [parent[-1:] for parent in record.parents]
1251
thunk_record = AdapterFactory(key, parents, record)
1252
path = self._mapper.map(prefix)
1253
# Note that this parses the file many times; we can do better but
1254
# as this only impacts weaves in terms of performance, it is
1256
vf = self._get_vf(path)
1257
vf.insert_record_stream([thunk_record])
1259
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1260
"""Iterate over the lines in the versioned files from keys.
1262
This may return lines from other keys. Each item the returned
1263
iterator yields is a tuple of a line and a text version that that line
1264
is present in (not introduced in).
1266
Ordering of results is in whatever order is most suitable for the
1267
underlying storage format.
1269
If a progress bar is supplied, it may be used to indicate progress.
1270
The caller is responsible for cleaning up progress bars (because this
1274
* Lines are normalised by the underlying store: they will all have \n
1276
* Lines are returned in arbitrary order.
1278
:return: An iterator over (line, key).
1280
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1281
for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1282
yield line, prefix + (version,)
1284
def _iter_all_components(self):
1285
for path, prefix in self._get_all_prefixes():
1286
yield prefix, self._get_vf(path)
1289
"""See VersionedFiles.keys()."""
1291
for prefix, vf in self._iter_all_components():
1292
for suffix in vf.versions():
1293
result.add(prefix + (suffix,))
1297
class _PlanMergeVersionedFile(VersionedFiles):
1298
"""A VersionedFile for uncommitted and committed texts.
1300
It is intended to allow merges to be planned with working tree texts.
1301
It implements only the small part of the VersionedFiles interface used by
1302
PlanMerge. It falls back to multiple versionedfiles for data not stored in
1303
_PlanMergeVersionedFile itself.
1305
:ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1306
queried for missing texts.
1309
def __init__(self, file_id):
1310
"""Create a _PlanMergeVersionedFile.
1312
:param file_id: Used with _PlanMerge code which is not yet fully
1313
tuple-keyspace aware.
1315
self._file_id = file_id
1316
# fallback locations
1317
self.fallback_versionedfiles = []
1318
# Parents for locally held keys.
1320
# line data for locally held keys.
1322
# key lookup providers
1323
self._providers = [DictParentsProvider(self._parents)]
1325
def plan_merge(self, ver_a, ver_b, base=None):
1326
"""See VersionedFile.plan_merge"""
1327
from bzrlib.merge import _PlanMerge
1329
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1330
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1331
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1332
return _PlanMerge._subtract_plans(old_plan, new_plan)
1334
def plan_lca_merge(self, ver_a, ver_b, base=None):
1335
from bzrlib.merge import _PlanLCAMerge
1337
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1340
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1341
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1343
def add_lines(self, key, parents, lines):
1344
"""See VersionedFiles.add_lines
1346
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1347
are permitted. Only reserved ids are permitted.
1349
if type(key) is not tuple:
1350
raise TypeError(key)
1351
if not revision.is_reserved_id(key[-1]):
1352
raise ValueError('Only reserved ids may be used')
1354
raise ValueError('Parents may not be None')
1356
raise ValueError('Lines may not be None')
1357
self._parents[key] = tuple(parents)
1358
self._lines[key] = lines
1360
def get_record_stream(self, keys, ordering, include_delta_closure):
1363
if key in self._lines:
1364
lines = self._lines[key]
1365
parents = self._parents[key]
1367
yield ChunkedContentFactory(key, parents, None, lines)
1368
for versionedfile in self.fallback_versionedfiles:
1369
for record in versionedfile.get_record_stream(
1370
pending, 'unordered', True):
1371
if record.storage_kind == 'absent':
1374
pending.remove(record.key)
1378
# report absent entries
1380
yield AbsentContentFactory(key)
1382
def get_parent_map(self, keys):
1383
"""See VersionedFiles.get_parent_map"""
1384
# We create a new provider because a fallback may have been added.
1385
# If we make fallbacks private we can update a stack list and avoid
1386
# object creation thrashing.
1389
if revision.NULL_REVISION in keys:
1390
keys.remove(revision.NULL_REVISION)
1391
result[revision.NULL_REVISION] = ()
1392
self._providers = self._providers[:1] + self.fallback_versionedfiles
1394
StackedParentsProvider(self._providers).get_parent_map(keys))
1395
for key, parents in result.iteritems():
1397
result[key] = (revision.NULL_REVISION,)
1401
class PlanWeaveMerge(TextMerge):
1402
"""Weave merge that takes a plan as its input.
1404
This exists so that VersionedFile.plan_merge is implementable.
1405
Most callers will want to use WeaveMerge instead.
1408
def __init__(self, plan, a_marker=TextMerge.A_MARKER,
1409
b_marker=TextMerge.B_MARKER):
1410
TextMerge.__init__(self, a_marker, b_marker)
1413
def _merge_struct(self):
1418
def outstanding_struct():
1419
if not lines_a and not lines_b:
1421
elif ch_a and not ch_b:
1424
elif ch_b and not ch_a:
1426
elif lines_a == lines_b:
1429
yield (lines_a, lines_b)
1431
# We previously considered either 'unchanged' or 'killed-both' lines
1432
# to be possible places to resynchronize. However, assuming agreement
1433
# on killed-both lines may be too aggressive. -- mbp 20060324
1434
for state, line in self.plan:
1435
if state == 'unchanged':
1436
# resync and flush queued conflicts changes if any
1437
for struct in outstanding_struct():
1443
if state == 'unchanged':
1446
elif state == 'killed-a':
1448
lines_b.append(line)
1449
elif state == 'killed-b':
1451
lines_a.append(line)
1452
elif state == 'new-a':
1454
lines_a.append(line)
1455
elif state == 'new-b':
1457
lines_b.append(line)
1458
elif state == 'conflicted-a':
1460
lines_a.append(line)
1461
elif state == 'conflicted-b':
1463
lines_b.append(line)
1464
elif state == 'killed-both':
1465
# This counts as a change, even though there is no associated
1469
if state not in ('irrelevant', 'ghost-a', 'ghost-b',
1471
raise AssertionError(state)
1472
for struct in outstanding_struct():
1476
class WeaveMerge(PlanWeaveMerge):
1477
"""Weave merge that takes a VersionedFile and two versions as its input."""
1479
def __init__(self, versionedfile, ver_a, ver_b,
1480
a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1481
plan = versionedfile.plan_merge(ver_a, ver_b)
1482
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1485
class VirtualVersionedFiles(VersionedFiles):
1486
"""Dummy implementation for VersionedFiles that uses other functions for
1487
obtaining fulltexts and parent maps.
1489
This is always on the bottom of the stack and uses string keys
1490
(rather than tuples) internally.
1493
def __init__(self, get_parent_map, get_lines):
1494
"""Create a VirtualVersionedFiles.
1496
:param get_parent_map: Same signature as Repository.get_parent_map.
1497
:param get_lines: Should return lines for specified key or None if
1500
super(VirtualVersionedFiles, self).__init__()
1501
self._get_parent_map = get_parent_map
1502
self._get_lines = get_lines
1504
def check(self, progressbar=None):
1505
"""See VersionedFiles.check.
1507
:note: Always returns True for VirtualVersionedFiles.
1511
def add_mpdiffs(self, records):
1512
"""See VersionedFiles.mpdiffs.
1514
:note: Not implemented for VirtualVersionedFiles.
1516
raise NotImplementedError(self.add_mpdiffs)
1518
def get_parent_map(self, keys):
1519
"""See VersionedFiles.get_parent_map."""
1520
return dict([((k,), tuple([(p,) for p in v]))
1521
for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
1523
def get_sha1s(self, keys):
1524
"""See VersionedFiles.get_sha1s."""
1527
lines = self._get_lines(k)
1528
if lines is not None:
1529
if not isinstance(lines, list):
1530
raise AssertionError
1531
ret[(k,)] = osutils.sha_strings(lines)
1534
def get_record_stream(self, keys, ordering, include_delta_closure):
1535
"""See VersionedFiles.get_record_stream."""
1536
for (k,) in list(keys):
1537
lines = self._get_lines(k)
1538
if lines is not None:
1539
if not isinstance(lines, list):
1540
raise AssertionError
1541
yield ChunkedContentFactory((k,), None,
1542
sha1=osutils.sha_strings(lines),
1545
yield AbsentContentFactory((k,))
1547
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1548
"""See VersionedFile.iter_lines_added_or_present_in_versions()."""
1549
for i, (key,) in enumerate(keys):
1551
pb.update("Finding changed lines", i, len(keys))
1552
for l in self._get_lines(key):
1556
def network_bytes_to_kind_and_offset(network_bytes):
1557
"""Strip of a record kind from the front of network_bytes.
1559
:param network_bytes: The bytes of a record.
1560
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1562
line_end = network_bytes.find('\n')
1563
storage_kind = network_bytes[:line_end]
1564
return storage_kind, line_end + 1
1567
class NetworkRecordStream(object):
1568
"""A record_stream which reconstitures a serialised stream."""
1570
def __init__(self, bytes_iterator):
1571
"""Create a NetworkRecordStream.
1573
:param bytes_iterator: An iterator of bytes. Each item in this
1574
iterator should have been obtained from a record_streams'
1575
record.get_bytes_as(record.storage_kind) call.
1577
self._bytes_iterator = bytes_iterator
1578
self._kind_factory = {
1579
'fulltext': fulltext_network_to_record,
1580
'groupcompress-block': groupcompress.network_block_to_records,
1581
'knit-ft-gz': knit.knit_network_to_record,
1582
'knit-delta-gz': knit.knit_network_to_record,
1583
'knit-annotated-ft-gz': knit.knit_network_to_record,
1584
'knit-annotated-delta-gz': knit.knit_network_to_record,
1585
'knit-delta-closure': knit.knit_delta_closure_to_records,
1591
:return: An iterator as per VersionedFiles.get_record_stream().
1593
for bytes in self._bytes_iterator:
1594
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1595
for record in self._kind_factory[storage_kind](
1596
storage_kind, bytes, line_end):
1600
def fulltext_network_to_record(kind, bytes, line_end):
1601
"""Convert a network fulltext record to record."""
1602
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1603
record_meta = bytes[line_end+4:line_end+4+meta_len]
1604
key, parents = bencode.bdecode_as_tuple(record_meta)
1605
if parents == 'nil':
1607
fulltext = bytes[line_end+4+meta_len:]
1608
return [FulltextContentFactory(key, parents, None, fulltext)]
1611
def _length_prefix(bytes):
1612
return struct.pack('!L', len(bytes))
1615
def record_to_fulltext_bytes(record):
1616
if record.parents is None:
1619
parents = record.parents
1620
record_meta = bencode.bencode((record.key, parents))
1621
record_content = record.get_bytes_as('fulltext')
1622
return "fulltext\n%s%s%s" % (
1623
_length_prefix(record_meta), record_meta, record_content)
1626
def sort_groupcompress(parent_map):
1627
"""Sort and group the keys in parent_map into groupcompress order.
1629
groupcompress is defined (currently) as reverse-topological order, grouped
1632
:return: A sorted-list of keys
1634
# gc-optimal ordering is approximately reverse topological,
1635
# properly grouped by file-id.
1637
for item in parent_map.iteritems():
1639
if isinstance(key, str) or len(key) == 1:
1644
per_prefix_map[prefix].append(item)
1646
per_prefix_map[prefix] = [item]
1649
for prefix in sorted(per_prefix_map):
1650
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
added
removed
bzrlib/versionedfile.py
