1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20
"""Versioned text file storage api."""
23
from cStringIO import StringIO
26
from zlib import adler32
28
from bzrlib.lazy_import import lazy_import
29
lazy_import(globals(), """
44
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
45
from bzrlib.transport.memory import MemoryTransport
47
from bzrlib.inter import InterObject
48
from bzrlib.registry import Registry
49
from bzrlib.symbol_versioning import *
50
from bzrlib.textmerge import TextMerge
51
from bzrlib import bencode
54
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
56
'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
59
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
'bzrlib.knit', 'DeltaAnnotatedToFullText')
63
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
64
'bzrlib.knit', 'FTAnnotatedToUnannotated')
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
'bzrlib.knit', 'FTAnnotatedToFullText')
67
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
# 'bzrlib.knit', 'FTAnnotatedToChunked')
71
class ContentFactory(object):
72
"""Abstract interface for insertion and retrieval from a VersionedFile.
74
:ivar sha1: None, or the sha1 of the content fulltext.
75
:ivar storage_kind: The native storage kind of this factory. One of
76
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
77
'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
78
'knit-annotated-delta-gz', 'knit-ft-gz', 'knit-delta-gz'.
79
:ivar key: The key of this content. Each key is a tuple with a single
81
:ivar parents: A tuple of parent keys for self.key. If the object has
82
no parent information, None (as opposed to () for an empty list of
87
"""Create a ContentFactory."""
89
self.storage_kind = None
94
class ChunkedContentFactory(ContentFactory):
95
"""Static data content factory.
97
This takes a 'chunked' list of strings. The only requirement on 'chunked' is
98
that ''.join(lines) becomes a valid fulltext. A tuple of a single string
99
satisfies this, as does a list of lines.
101
:ivar sha1: None, or the sha1 of the content fulltext.
102
:ivar storage_kind: The native storage kind of this factory. Always
104
:ivar key: The key of this content. Each key is a tuple with a single
106
:ivar parents: A tuple of parent keys for self.key. If the object has
107
no parent information, None (as opposed to () for an empty list of
111
def __init__(self, key, parents, sha1, chunks):
112
"""Create a ContentFactory."""
114
self.storage_kind = 'chunked'
116
self.parents = parents
117
self._chunks = chunks
119
def get_bytes_as(self, storage_kind):
120
if storage_kind == 'chunked':
122
elif storage_kind == 'fulltext':
123
return ''.join(self._chunks)
124
raise errors.UnavailableRepresentation(self.key, storage_kind,
128
class FulltextContentFactory(ContentFactory):
129
"""Static data content factory.
131
This takes a fulltext when created and just returns that during
132
get_bytes_as('fulltext').
134
:ivar sha1: None, or the sha1 of the content fulltext.
135
:ivar storage_kind: The native storage kind of this factory. Always
137
:ivar key: The key of this content. Each key is a tuple with a single
139
:ivar parents: A tuple of parent keys for self.key. If the object has
140
no parent information, None (as opposed to () for an empty list of
144
def __init__(self, key, parents, sha1, text):
145
"""Create a ContentFactory."""
147
self.storage_kind = 'fulltext'
149
self.parents = parents
152
def get_bytes_as(self, storage_kind):
153
if storage_kind == self.storage_kind:
155
elif storage_kind == 'chunked':
157
raise errors.UnavailableRepresentation(self.key, storage_kind,
161
class AbsentContentFactory(ContentFactory):
162
"""A placeholder content factory for unavailable texts.
165
:ivar storage_kind: 'absent'.
166
:ivar key: The key of this content. Each key is a tuple with a single
171
def __init__(self, key):
172
"""Create a ContentFactory."""
174
self.storage_kind = 'absent'
178
def get_bytes_as(self, storage_kind):
179
raise ValueError('A request was made for key: %s, but that'
180
' content is not available, and the calling'
181
' code does not handle if it is missing.'
185
class AdapterFactory(ContentFactory):
186
"""A content factory to adapt between key prefix's."""
188
def __init__(self, key, parents, adapted):
189
"""Create an adapter factory instance."""
191
self.parents = parents
192
self._adapted = adapted
194
def __getattr__(self, attr):
195
"""Return a member from the adapted object."""
196
if attr in ('key', 'parents'):
197
return self.__dict__[attr]
199
return getattr(self._adapted, attr)
202
def filter_absent(record_stream):
203
"""Adapt a record stream to remove absent records."""
204
for record in record_stream:
205
if record.storage_kind != 'absent':
209
class VersionedFile(object):
210
"""Versioned text file storage.
212
A versioned file manages versions of line-based text files,
213
keeping track of the originating version for each line.
215
To clients the "lines" of the file are represented as a list of
216
strings. These strings will typically have terminal newline
217
characters, but this is not required. In particular files commonly
218
do not have a newline at the end of the file.
220
Texts are identified by a version-id string.
224
def check_not_reserved_id(version_id):
225
revision.check_not_reserved_id(version_id)
227
def copy_to(self, name, transport):
228
"""Copy this versioned file to name on transport."""
229
raise NotImplementedError(self.copy_to)
231
def get_record_stream(self, versions, ordering, include_delta_closure):
232
"""Get a stream of records for versions.
234
:param versions: The versions to include. Each version is a tuple
236
:param ordering: Either 'unordered' or 'topological'. A topologically
237
sorted stream has compression parents strictly before their
239
:param include_delta_closure: If True then the closure across any
240
compression parents will be included (in the data content of the
241
stream, not in the emitted records). This guarantees that
242
'fulltext' can be used successfully on every record.
243
:return: An iterator of ContentFactory objects, each of which is only
244
valid until the iterator is advanced.
246
raise NotImplementedError(self.get_record_stream)
248
def has_version(self, version_id):
249
"""Returns whether version is present."""
250
raise NotImplementedError(self.has_version)
252
def insert_record_stream(self, stream):
253
"""Insert a record stream into this versioned file.
255
:param stream: A stream of records to insert.
257
:seealso VersionedFile.get_record_stream:
259
raise NotImplementedError
261
def add_lines(self, version_id, parents, lines, parent_texts=None,
262
left_matching_blocks=None, nostore_sha=None, random_id=False,
264
"""Add a single text on top of the versioned file.
266
Must raise RevisionAlreadyPresent if the new version is
267
already present in file history.
269
Must raise RevisionNotPresent if any of the given parents are
270
not present in file history.
272
:param lines: A list of lines. Each line must be a bytestring. And all
273
of them except the last must be terminated with \n and contain no
274
other \n's. The last line may either contain no \n's or a single
275
terminated \n. If the lines list does meet this constraint the add
276
routine may error or may succeed - but you will be unable to read
277
the data back accurately. (Checking the lines have been split
278
correctly is expensive and extremely unlikely to catch bugs so it
279
is not done at runtime unless check_content is True.)
280
:param parent_texts: An optional dictionary containing the opaque
281
representations of some or all of the parents of version_id to
282
allow delta optimisations. VERY IMPORTANT: the texts must be those
283
returned by add_lines or data corruption can be caused.
284
:param left_matching_blocks: a hint about which areas are common
285
between the text and its left-hand-parent. The format is
286
the SequenceMatcher.get_matching_blocks format.
287
:param nostore_sha: Raise ExistingContent and do not add the lines to
288
the versioned file if the digest of the lines matches this.
289
:param random_id: If True a random id has been selected rather than
290
an id determined by some deterministic process such as a converter
291
from a foreign VCS. When True the backend may choose not to check
292
for uniqueness of the resulting key within the versioned file, so
293
this should only be done when the result is expected to be unique
295
:param check_content: If True, the lines supplied are verified to be
296
bytestrings that are correctly formed lines.
297
:return: The text sha1, the number of bytes in the text, and an opaque
298
representation of the inserted version which can be provided
299
back to future add_lines calls in the parent_texts dictionary.
301
self._check_write_ok()
302
return self._add_lines(version_id, parents, lines, parent_texts,
303
left_matching_blocks, nostore_sha, random_id, check_content)
305
def _add_lines(self, version_id, parents, lines, parent_texts,
306
left_matching_blocks, nostore_sha, random_id, check_content):
307
"""Helper to do the class specific add_lines."""
308
raise NotImplementedError(self.add_lines)
310
def add_lines_with_ghosts(self, version_id, parents, lines,
311
parent_texts=None, nostore_sha=None, random_id=False,
312
check_content=True, left_matching_blocks=None):
313
"""Add lines to the versioned file, allowing ghosts to be present.
315
This takes the same parameters as add_lines and returns the same.
317
self._check_write_ok()
318
return self._add_lines_with_ghosts(version_id, parents, lines,
319
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
321
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
322
nostore_sha, random_id, check_content, left_matching_blocks):
323
"""Helper to do class specific add_lines_with_ghosts."""
324
raise NotImplementedError(self.add_lines_with_ghosts)
326
def check(self, progress_bar=None):
327
"""Check the versioned file for integrity."""
328
raise NotImplementedError(self.check)
330
def _check_lines_not_unicode(self, lines):
331
"""Check that lines being added to a versioned file are not unicode."""
333
if line.__class__ is not str:
334
raise errors.BzrBadParameterUnicode("lines")
336
def _check_lines_are_lines(self, lines):
337
"""Check that the lines really are full lines without inline EOL."""
339
if '\n' in line[:-1]:
340
raise errors.BzrBadParameterContainsNewline("lines")
342
def get_format_signature(self):
343
"""Get a text description of the data encoding in this file.
347
raise NotImplementedError(self.get_format_signature)
349
def make_mpdiffs(self, version_ids):
350
"""Create multiparent diffs for specified versions."""
351
knit_versions = set()
352
knit_versions.update(version_ids)
353
parent_map = self.get_parent_map(version_ids)
354
for version_id in version_ids:
356
knit_versions.update(parent_map[version_id])
358
raise errors.RevisionNotPresent(version_id, self)
359
# We need to filter out ghosts, because we can't diff against them.
360
knit_versions = set(self.get_parent_map(knit_versions).keys())
361
lines = dict(zip(knit_versions,
362
self._get_lf_split_line_list(knit_versions)))
364
for version_id in version_ids:
365
target = lines[version_id]
367
parents = [lines[p] for p in parent_map[version_id] if p in
370
# I don't know how this could ever trigger.
371
# parent_map[version_id] was already triggered in the previous
372
# for loop, and lines[p] has the 'if p in knit_versions' check,
373
# so we again won't have a KeyError.
374
raise errors.RevisionNotPresent(version_id, self)
376
left_parent_blocks = self._extract_blocks(version_id,
379
left_parent_blocks = None
380
diffs.append(multiparent.MultiParent.from_lines(target, parents,
384
def _extract_blocks(self, version_id, source, target):
387
def add_mpdiffs(self, records):
388
"""Add mpdiffs to this VersionedFile.
390
Records should be iterables of version, parents, expected_sha1,
391
mpdiff. mpdiff should be a MultiParent instance.
393
# Does this need to call self._check_write_ok()? (IanC 20070919)
395
mpvf = multiparent.MultiMemoryVersionedFile()
397
for version, parent_ids, expected_sha1, mpdiff in records:
398
versions.append(version)
399
mpvf.add_diff(mpdiff, version, parent_ids)
400
needed_parents = set()
401
for version, parent_ids, expected_sha1, mpdiff in records:
402
needed_parents.update(p for p in parent_ids
403
if not mpvf.has_version(p))
404
present_parents = set(self.get_parent_map(needed_parents).keys())
405
for parent_id, lines in zip(present_parents,
406
self._get_lf_split_line_list(present_parents)):
407
mpvf.add_version(lines, parent_id, [])
408
for (version, parent_ids, expected_sha1, mpdiff), lines in\
409
zip(records, mpvf.get_line_list(versions)):
410
if len(parent_ids) == 1:
411
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
412
mpvf.get_diff(parent_ids[0]).num_lines()))
414
left_matching_blocks = None
416
_, _, version_text = self.add_lines_with_ghosts(version,
417
parent_ids, lines, vf_parents,
418
left_matching_blocks=left_matching_blocks)
419
except NotImplementedError:
420
# The vf can't handle ghosts, so add lines normally, which will
421
# (reasonably) fail if there are ghosts in the data.
422
_, _, version_text = self.add_lines(version,
423
parent_ids, lines, vf_parents,
424
left_matching_blocks=left_matching_blocks)
425
vf_parents[version] = version_text
426
sha1s = self.get_sha1s(versions)
427
for version, parent_ids, expected_sha1, mpdiff in records:
428
if expected_sha1 != sha1s[version]:
429
raise errors.VersionedFileInvalidChecksum(version)
431
def get_text(self, version_id):
432
"""Return version contents as a text string.
434
Raises RevisionNotPresent if version is not present in
437
return ''.join(self.get_lines(version_id))
438
get_string = get_text
440
def get_texts(self, version_ids):
441
"""Return the texts of listed versions as a list of strings.
443
Raises RevisionNotPresent if version is not present in
446
return [''.join(self.get_lines(v)) for v in version_ids]
448
def get_lines(self, version_id):
449
"""Return version contents as a sequence of lines.
451
Raises RevisionNotPresent if version is not present in
454
raise NotImplementedError(self.get_lines)
456
def _get_lf_split_line_list(self, version_ids):
457
return [StringIO(t).readlines() for t in self.get_texts(version_ids)]
459
def get_ancestry(self, version_ids, topo_sorted=True):
460
"""Return a list of all ancestors of given version(s). This
461
will not include the null revision.
463
This list will not be topologically sorted if topo_sorted=False is
466
Must raise RevisionNotPresent if any of the given versions are
467
not present in file history."""
468
if isinstance(version_ids, basestring):
469
version_ids = [version_ids]
470
raise NotImplementedError(self.get_ancestry)
472
def get_ancestry_with_ghosts(self, version_ids):
473
"""Return a list of all ancestors of given version(s). This
474
will not include the null revision.
476
Must raise RevisionNotPresent if any of the given versions are
477
not present in file history.
479
Ghosts that are known about will be included in ancestry list,
480
but are not explicitly marked.
482
raise NotImplementedError(self.get_ancestry_with_ghosts)
484
def get_parent_map(self, version_ids):
485
"""Get a map of the parents of version_ids.
487
:param version_ids: The version ids to look up parents for.
488
:return: A mapping from version id to parents.
490
raise NotImplementedError(self.get_parent_map)
492
def get_parents_with_ghosts(self, version_id):
493
"""Return version names for parents of version_id.
495
Will raise RevisionNotPresent if version_id is not present
498
Ghosts that are known about will be included in the parent list,
499
but are not explicitly marked.
502
return list(self.get_parent_map([version_id])[version_id])
504
raise errors.RevisionNotPresent(version_id, self)
506
def annotate(self, version_id):
507
"""Return a list of (version-id, line) tuples for version_id.
509
:raise RevisionNotPresent: If the given version is
510
not present in file history.
512
raise NotImplementedError(self.annotate)
514
def iter_lines_added_or_present_in_versions(self, version_ids=None,
516
"""Iterate over the lines in the versioned file from version_ids.
518
This may return lines from other versions. Each item the returned
519
iterator yields is a tuple of a line and a text version that that line
520
is present in (not introduced in).
522
Ordering of results is in whatever order is most suitable for the
523
underlying storage format.
525
If a progress bar is supplied, it may be used to indicate progress.
526
The caller is responsible for cleaning up progress bars (because this
529
NOTES: Lines are normalised: they will all have \n terminators.
530
Lines are returned in arbitrary order.
532
:return: An iterator over (line, version_id).
534
raise NotImplementedError(self.iter_lines_added_or_present_in_versions)
536
def plan_merge(self, ver_a, ver_b):
537
"""Return pseudo-annotation indicating how the two versions merge.
539
This is computed between versions a and b and their common
542
Weave lines present in none of them are skipped entirely.
545
killed-base Dead in base revision
546
killed-both Killed in each revision
549
unchanged Alive in both a and b (possibly created in both)
552
ghost-a Killed in a, unborn in b
553
ghost-b Killed in b, unborn in a
554
irrelevant Not in either revision
556
raise NotImplementedError(VersionedFile.plan_merge)
558
def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
559
b_marker=TextMerge.B_MARKER):
560
return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
563
class RecordingVersionedFilesDecorator(object):
564
"""A minimal versioned files that records calls made on it.
566
Only enough methods have been added to support tests using it to date.
568
:ivar calls: A list of the calls made; can be reset at any time by
572
def __init__(self, backing_vf):
573
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
575
:param backing_vf: The versioned file to answer all methods.
577
self._backing_vf = backing_vf
580
def add_lines(self, key, parents, lines, parent_texts=None,
581
left_matching_blocks=None, nostore_sha=None, random_id=False,
583
self.calls.append(("add_lines", key, parents, lines, parent_texts,
584
left_matching_blocks, nostore_sha, random_id, check_content))
585
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
586
left_matching_blocks, nostore_sha, random_id, check_content)
589
self._backing_vf.check()
591
def get_parent_map(self, keys):
592
self.calls.append(("get_parent_map", copy(keys)))
593
return self._backing_vf.get_parent_map(keys)
595
def get_record_stream(self, keys, sort_order, include_delta_closure):
596
self.calls.append(("get_record_stream", list(keys), sort_order,
597
include_delta_closure))
598
return self._backing_vf.get_record_stream(keys, sort_order,
599
include_delta_closure)
601
def get_sha1s(self, keys):
602
self.calls.append(("get_sha1s", copy(keys)))
603
return self._backing_vf.get_sha1s(keys)
605
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
606
self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
607
return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
610
self.calls.append(("keys",))
611
return self._backing_vf.keys()
614
class OrderingVersionedFilesDecorator(RecordingVersionedFilesDecorator):
615
"""A VF that records calls, and returns keys in specific order.
617
:ivar calls: A list of the calls made; can be reset at any time by
621
def __init__(self, backing_vf, key_priority):
622
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
624
:param backing_vf: The versioned file to answer all methods.
625
:param key_priority: A dictionary defining what order keys should be
626
returned from an 'unordered' get_record_stream request.
627
Keys with lower priority are returned first, keys not present in
628
the map get an implicit priority of 0, and are returned in
629
lexicographical order.
631
RecordingVersionedFilesDecorator.__init__(self, backing_vf)
632
self._key_priority = key_priority
634
def get_record_stream(self, keys, sort_order, include_delta_closure):
635
self.calls.append(("get_record_stream", list(keys), sort_order,
636
include_delta_closure))
637
if sort_order == 'unordered':
639
return (self._key_priority.get(key, 0), key)
640
# Use a defined order by asking for the keys one-by-one from the
642
for key in sorted(keys, key=sort_key):
643
for record in self._backing_vf.get_record_stream([key],
644
'unordered', include_delta_closure):
647
for record in self._backing_vf.get_record_stream(keys, sort_order,
648
include_delta_closure):
652
class KeyMapper(object):
653
"""KeyMappers map between keys and underlying partitioned storage."""
656
"""Map key to an underlying storage identifier.
658
:param key: A key tuple e.g. ('file-id', 'revision-id').
659
:return: An underlying storage identifier, specific to the partitioning
662
raise NotImplementedError(self.map)
664
def unmap(self, partition_id):
665
"""Map a partitioned storage id back to a key prefix.
667
:param partition_id: The underlying partition id.
668
:return: As much of a key (or prefix) as is derivable from the partition
671
raise NotImplementedError(self.unmap)
674
class ConstantMapper(KeyMapper):
675
"""A key mapper that maps to a constant result."""
677
def __init__(self, result):
678
"""Create a ConstantMapper which will return result for all maps."""
679
self._result = result
682
"""See KeyMapper.map()."""
686
class URLEscapeMapper(KeyMapper):
687
"""Base class for use with transport backed storage.
689
This provides a map and unmap wrapper that respectively url escape and
690
unescape their outputs and inputs.
694
"""See KeyMapper.map()."""
695
return urllib.quote(self._map(key))
697
def unmap(self, partition_id):
698
"""See KeyMapper.unmap()."""
699
return self._unmap(urllib.unquote(partition_id))
702
class PrefixMapper(URLEscapeMapper):
703
"""A key mapper that extracts the first component of a key.
705
This mapper is for use with a transport based backend.
709
"""See KeyMapper.map()."""
712
def _unmap(self, partition_id):
713
"""See KeyMapper.unmap()."""
714
return (partition_id,)
717
class HashPrefixMapper(URLEscapeMapper):
718
"""A key mapper that combines the first component of a key with a hash.
720
This mapper is for use with a transport based backend.
724
"""See KeyMapper.map()."""
725
prefix = self._escape(key[0])
726
return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
728
def _escape(self, prefix):
729
"""No escaping needed here."""
732
def _unmap(self, partition_id):
733
"""See KeyMapper.unmap()."""
734
return (self._unescape(osutils.basename(partition_id)),)
736
def _unescape(self, basename):
737
"""No unescaping needed for HashPrefixMapper."""
741
class HashEscapedPrefixMapper(HashPrefixMapper):
742
"""Combines the escaped first component of a key with a hash.
744
This mapper is for use with a transport based backend.
747
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
749
def _escape(self, prefix):
750
"""Turn a key element into a filesystem safe string.
752
This is similar to a plain urllib.quote, except
753
it uses specific safe characters, so that it doesn't
754
have to translate a lot of valid file ids.
756
# @ does not get escaped. This is because it is a valid
757
# filesystem character we use all the time, and it looks
758
# a lot better than seeing %40 all the time.
759
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
763
def _unescape(self, basename):
764
"""Escaped names are easily unescaped by urlutils."""
765
return urllib.unquote(basename)
768
def make_versioned_files_factory(versioned_file_factory, mapper):
769
"""Create a ThunkedVersionedFiles factory.
771
This will create a callable which when called creates a
772
ThunkedVersionedFiles on a transport, using mapper to access individual
773
versioned files, and versioned_file_factory to create each individual file.
775
def factory(transport):
776
return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
781
class VersionedFiles(object):
782
"""Storage for many versioned files.
784
This object allows a single keyspace for accessing the history graph and
785
contents of named bytestrings.
787
Currently no implementation allows the graph of different key prefixes to
788
intersect, but the API does allow such implementations in the future.
790
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
791
may have a different length key-size, but that size will be constant for
792
all texts added to or retrieved from it. For instance, bzrlib uses
793
instances with a key-size of 2 for storing user files in a repository, with
794
the first element the fileid, and the second the version of that file.
796
The use of tuples allows a single code base to support several different
797
uses with only the mapping logic changing from instance to instance.
800
def add_lines(self, key, parents, lines, parent_texts=None,
801
left_matching_blocks=None, nostore_sha=None, random_id=False,
803
"""Add a text to the store.
805
:param key: The key tuple of the text to add. If the last element is
806
None, a CHK string will be generated during the addition.
807
:param parents: The parents key tuples of the text to add.
808
:param lines: A list of lines. Each line must be a bytestring. And all
809
of them except the last must be terminated with \n and contain no
810
other \n's. The last line may either contain no \n's or a single
811
terminating \n. If the lines list does meet this constraint the add
812
routine may error or may succeed - but you will be unable to read
813
the data back accurately. (Checking the lines have been split
814
correctly is expensive and extremely unlikely to catch bugs so it
815
is not done at runtime unless check_content is True.)
816
:param parent_texts: An optional dictionary containing the opaque
817
representations of some or all of the parents of version_id to
818
allow delta optimisations. VERY IMPORTANT: the texts must be those
819
returned by add_lines or data corruption can be caused.
820
:param left_matching_blocks: a hint about which areas are common
821
between the text and its left-hand-parent. The format is
822
the SequenceMatcher.get_matching_blocks format.
823
:param nostore_sha: Raise ExistingContent and do not add the lines to
824
the versioned file if the digest of the lines matches this.
825
:param random_id: If True a random id has been selected rather than
826
an id determined by some deterministic process such as a converter
827
from a foreign VCS. When True the backend may choose not to check
828
for uniqueness of the resulting key within the versioned file, so
829
this should only be done when the result is expected to be unique
831
:param check_content: If True, the lines supplied are verified to be
832
bytestrings that are correctly formed lines.
833
:return: The text sha1, the number of bytes in the text, and an opaque
834
representation of the inserted version which can be provided
835
back to future add_lines calls in the parent_texts dictionary.
837
raise NotImplementedError(self.add_lines)
839
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
840
"""Add a text to the store.
842
This is a private function for use by CommitBuilder.
844
:param key: The key tuple of the text to add. If the last element is
845
None, a CHK string will be generated during the addition.
846
:param parents: The parents key tuples of the text to add.
847
:param text: A string containing the text to be committed.
848
:param nostore_sha: Raise ExistingContent and do not add the lines to
849
the versioned file if the digest of the lines matches this.
850
:param random_id: If True a random id has been selected rather than
851
an id determined by some deterministic process such as a converter
852
from a foreign VCS. When True the backend may choose not to check
853
for uniqueness of the resulting key within the versioned file, so
854
this should only be done when the result is expected to be unique
856
:param check_content: If True, the lines supplied are verified to be
857
bytestrings that are correctly formed lines.
858
:return: The text sha1, the number of bytes in the text, and an opaque
859
representation of the inserted version which can be provided
860
back to future _add_text calls in the parent_texts dictionary.
862
# The default implementation just thunks over to .add_lines(),
863
# inefficient, but it works.
864
return self.add_lines(key, parents, osutils.split_lines(text),
865
nostore_sha=nostore_sha,
869
def add_mpdiffs(self, records):
870
"""Add mpdiffs to this VersionedFile.
872
Records should be iterables of version, parents, expected_sha1,
873
mpdiff. mpdiff should be a MultiParent instance.
876
mpvf = multiparent.MultiMemoryVersionedFile()
878
for version, parent_ids, expected_sha1, mpdiff in records:
879
versions.append(version)
880
mpvf.add_diff(mpdiff, version, parent_ids)
881
needed_parents = set()
882
for version, parent_ids, expected_sha1, mpdiff in records:
883
needed_parents.update(p for p in parent_ids
884
if not mpvf.has_version(p))
885
# It seems likely that adding all the present parents as fulltexts can
886
# easily exhaust memory.
887
chunks_to_lines = osutils.chunks_to_lines
888
for record in self.get_record_stream(needed_parents, 'unordered',
890
if record.storage_kind == 'absent':
892
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
894
for (key, parent_keys, expected_sha1, mpdiff), lines in\
895
zip(records, mpvf.get_line_list(versions)):
896
if len(parent_keys) == 1:
897
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
898
mpvf.get_diff(parent_keys[0]).num_lines()))
900
left_matching_blocks = None
901
version_sha1, _, version_text = self.add_lines(key,
902
parent_keys, lines, vf_parents,
903
left_matching_blocks=left_matching_blocks)
904
if version_sha1 != expected_sha1:
905
raise errors.VersionedFileInvalidChecksum(version)
906
vf_parents[key] = version_text
908
def annotate(self, key):
909
"""Return a list of (version-key, line) tuples for the text of key.
911
:raise RevisionNotPresent: If the key is not present.
913
raise NotImplementedError(self.annotate)
915
def check(self, progress_bar=None):
916
"""Check this object for integrity."""
917
raise NotImplementedError(self.check)
920
def check_not_reserved_id(version_id):
921
revision.check_not_reserved_id(version_id)
923
def _check_lines_not_unicode(self, lines):
924
"""Check that lines being added to a versioned file are not unicode."""
926
if line.__class__ is not str:
927
raise errors.BzrBadParameterUnicode("lines")
929
def _check_lines_are_lines(self, lines):
930
"""Check that the lines really are full lines without inline EOL."""
932
if '\n' in line[:-1]:
933
raise errors.BzrBadParameterContainsNewline("lines")
935
def get_parent_map(self, keys):
936
"""Get a map of the parents of keys.
938
:param keys: The keys to look up parents for.
939
:return: A mapping from keys to parents. Absent keys are absent from
942
raise NotImplementedError(self.get_parent_map)
944
def get_record_stream(self, keys, ordering, include_delta_closure):
945
"""Get a stream of records for keys.
947
:param keys: The keys to include.
948
:param ordering: Either 'unordered' or 'topological'. A topologically
949
sorted stream has compression parents strictly before their
951
:param include_delta_closure: If True then the closure across any
952
compression parents will be included (in the opaque data).
953
:return: An iterator of ContentFactory objects, each of which is only
954
valid until the iterator is advanced.
956
raise NotImplementedError(self.get_record_stream)
958
def get_sha1s(self, keys):
959
"""Get the sha1's of the texts for the given keys.
961
:param keys: The names of the keys to lookup
962
:return: a dict from key to sha1 digest. Keys of texts which are not
963
present in the store are not present in the returned
966
raise NotImplementedError(self.get_sha1s)
968
has_key = index._has_key_from_parent_map
970
def get_missing_compression_parent_keys(self):
971
"""Return an iterable of keys of missing compression parents.
973
Check this after calling insert_record_stream to find out if there are
974
any missing compression parents. If there are, the records that
975
depend on them are not able to be inserted safely. The precise
976
behaviour depends on the concrete VersionedFiles class in use.
978
Classes that do not support this will raise NotImplementedError.
980
raise NotImplementedError(self.get_missing_compression_parent_keys)
982
def insert_record_stream(self, stream):
983
"""Insert a record stream into this container.
985
:param stream: A stream of records to insert.
987
:seealso VersionedFile.get_record_stream:
989
raise NotImplementedError
991
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
992
"""Iterate over the lines in the versioned files from keys.
994
This may return lines from other keys. Each item the returned
995
iterator yields is a tuple of a line and a text version that that line
996
is present in (not introduced in).
998
Ordering of results is in whatever order is most suitable for the
999
underlying storage format.
1001
If a progress bar is supplied, it may be used to indicate progress.
1002
The caller is responsible for cleaning up progress bars (because this
1006
* Lines are normalised by the underlying store: they will all have \n
1008
* Lines are returned in arbitrary order.
1010
:return: An iterator over (line, key).
1012
raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
1015
"""Return a iterable of the keys for all the contained texts."""
1016
raise NotImplementedError(self.keys)
1018
def make_mpdiffs(self, keys):
1019
"""Create multiparent diffs for specified keys."""
1020
keys_order = tuple(keys)
1021
keys = frozenset(keys)
1022
knit_keys = set(keys)
1023
parent_map = self.get_parent_map(keys)
1024
for parent_keys in parent_map.itervalues():
1026
knit_keys.update(parent_keys)
1027
missing_keys = keys - set(parent_map)
1029
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1030
# We need to filter out ghosts, because we can't diff against them.
1031
maybe_ghosts = knit_keys - keys
1032
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1033
knit_keys.difference_update(ghosts)
1035
chunks_to_lines = osutils.chunks_to_lines
1036
for record in self.get_record_stream(knit_keys, 'topological', True):
1037
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1038
# line_block_dict = {}
1039
# for parent, blocks in record.extract_line_blocks():
1040
# line_blocks[parent] = blocks
1041
# line_blocks[record.key] = line_block_dict
1043
for key in keys_order:
1045
parents = parent_map[key] or []
1046
# Note that filtering knit_keys can lead to a parent difference
1047
# between the creation and the application of the mpdiff.
1048
parent_lines = [lines[p] for p in parents if p in knit_keys]
1049
if len(parent_lines) > 0:
1050
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1053
left_parent_blocks = None
1054
diffs.append(multiparent.MultiParent.from_lines(target,
1055
parent_lines, left_parent_blocks))
1058
missing_keys = index._missing_keys_from_parent_map
1060
def _extract_blocks(self, version_id, source, target):
1064
class ThunkedVersionedFiles(VersionedFiles):
1065
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1067
This object allows a single keyspace for accessing the history graph and
1068
contents of named bytestrings.
1070
Currently no implementation allows the graph of different key prefixes to
1071
intersect, but the API does allow such implementations in the future.
1074
def __init__(self, transport, file_factory, mapper, is_locked):
1075
"""Create a ThunkedVersionedFiles."""
1076
self._transport = transport
1077
self._file_factory = file_factory
1078
self._mapper = mapper
1079
self._is_locked = is_locked
1081
def add_lines(self, key, parents, lines, parent_texts=None,
1082
left_matching_blocks=None, nostore_sha=None, random_id=False,
1083
check_content=True):
1084
"""See VersionedFiles.add_lines()."""
1085
path = self._mapper.map(key)
1086
version_id = key[-1]
1087
parents = [parent[-1] for parent in parents]
1088
vf = self._get_vf(path)
1091
return vf.add_lines_with_ghosts(version_id, parents, lines,
1092
parent_texts=parent_texts,
1093
left_matching_blocks=left_matching_blocks,
1094
nostore_sha=nostore_sha, random_id=random_id,
1095
check_content=check_content)
1096
except NotImplementedError:
1097
return vf.add_lines(version_id, parents, lines,
1098
parent_texts=parent_texts,
1099
left_matching_blocks=left_matching_blocks,
1100
nostore_sha=nostore_sha, random_id=random_id,
1101
check_content=check_content)
1102
except errors.NoSuchFile:
1103
# parent directory may be missing, try again.
1104
self._transport.mkdir(osutils.dirname(path))
1106
return vf.add_lines_with_ghosts(version_id, parents, lines,
1107
parent_texts=parent_texts,
1108
left_matching_blocks=left_matching_blocks,
1109
nostore_sha=nostore_sha, random_id=random_id,
1110
check_content=check_content)
1111
except NotImplementedError:
1112
return vf.add_lines(version_id, parents, lines,
1113
parent_texts=parent_texts,
1114
left_matching_blocks=left_matching_blocks,
1115
nostore_sha=nostore_sha, random_id=random_id,
1116
check_content=check_content)
1118
def annotate(self, key):
1119
"""Return a list of (version-key, line) tuples for the text of key.
1121
:raise RevisionNotPresent: If the key is not present.
1124
path = self._mapper.map(prefix)
1125
vf = self._get_vf(path)
1126
origins = vf.annotate(key[-1])
1128
for origin, line in origins:
1129
result.append((prefix + (origin,), line))
1132
def get_annotator(self):
1133
return annotate.Annotator(self)
1135
def check(self, progress_bar=None):
1136
"""See VersionedFiles.check()."""
1137
for prefix, vf in self._iter_all_components():
1140
def get_parent_map(self, keys):
1141
"""Get a map of the parents of keys.
1143
:param keys: The keys to look up parents for.
1144
:return: A mapping from keys to parents. Absent keys are absent from
1147
prefixes = self._partition_keys(keys)
1149
for prefix, suffixes in prefixes.items():
1150
path = self._mapper.map(prefix)
1151
vf = self._get_vf(path)
1152
parent_map = vf.get_parent_map(suffixes)
1153
for key, parents in parent_map.items():
1154
result[prefix + (key,)] = tuple(
1155
prefix + (parent,) for parent in parents)
1158
def _get_vf(self, path):
1159
if not self._is_locked():
1160
raise errors.ObjectNotLocked(self)
1161
return self._file_factory(path, self._transport, create=True,
1162
get_scope=lambda:None)
1164
def _partition_keys(self, keys):
1165
"""Turn keys into a dict of prefix:suffix_list."""
1168
prefix_keys = result.setdefault(key[:-1], [])
1169
prefix_keys.append(key[-1])
1172
def _get_all_prefixes(self):
1173
# Identify all key prefixes.
1174
# XXX: A bit hacky, needs polish.
1175
if type(self._mapper) == ConstantMapper:
1176
paths = [self._mapper.map(())]
1180
for quoted_relpath in self._transport.iter_files_recursive():
1181
path, ext = os.path.splitext(quoted_relpath)
1183
paths = list(relpaths)
1184
prefixes = [self._mapper.unmap(path) for path in paths]
1185
return zip(paths, prefixes)
1187
def get_record_stream(self, keys, ordering, include_delta_closure):
1188
"""See VersionedFiles.get_record_stream()."""
1189
# Ordering will be taken care of by each partitioned store; group keys
1192
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1193
suffixes = [(suffix,) for suffix in suffixes]
1194
for record in vf.get_record_stream(suffixes, ordering,
1195
include_delta_closure):
1196
if record.parents is not None:
1197
record.parents = tuple(
1198
prefix + parent for parent in record.parents)
1199
record.key = prefix + record.key
1202
def _iter_keys_vf(self, keys):
1203
prefixes = self._partition_keys(keys)
1205
for prefix, suffixes in prefixes.items():
1206
path = self._mapper.map(prefix)
1207
vf = self._get_vf(path)
1208
yield prefix, suffixes, vf
1210
def get_sha1s(self, keys):
1211
"""See VersionedFiles.get_sha1s()."""
1213
for prefix,suffixes, vf in self._iter_keys_vf(keys):
1214
vf_sha1s = vf.get_sha1s(suffixes)
1215
for suffix, sha1 in vf_sha1s.iteritems():
1216
sha1s[prefix + (suffix,)] = sha1
1219
def insert_record_stream(self, stream):
1220
"""Insert a record stream into this container.
1222
:param stream: A stream of records to insert.
1224
:seealso VersionedFile.get_record_stream:
1226
for record in stream:
1227
prefix = record.key[:-1]
1228
key = record.key[-1:]
1229
if record.parents is not None:
1230
parents = [parent[-1:] for parent in record.parents]
1233
thunk_record = AdapterFactory(key, parents, record)
1234
path = self._mapper.map(prefix)
1235
# Note that this parses the file many times; we can do better but
1236
# as this only impacts weaves in terms of performance, it is
1238
vf = self._get_vf(path)
1239
vf.insert_record_stream([thunk_record])
1241
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1242
"""Iterate over the lines in the versioned files from keys.
1244
This may return lines from other keys. Each item the returned
1245
iterator yields is a tuple of a line and a text version that that line
1246
is present in (not introduced in).
1248
Ordering of results is in whatever order is most suitable for the
1249
underlying storage format.
1251
If a progress bar is supplied, it may be used to indicate progress.
1252
The caller is responsible for cleaning up progress bars (because this
1256
* Lines are normalised by the underlying store: they will all have \n
1258
* Lines are returned in arbitrary order.
1260
:return: An iterator over (line, key).
1262
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1263
for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1264
yield line, prefix + (version,)
1266
def _iter_all_components(self):
1267
for path, prefix in self._get_all_prefixes():
1268
yield prefix, self._get_vf(path)
1271
"""See VersionedFiles.keys()."""
1273
for prefix, vf in self._iter_all_components():
1274
for suffix in vf.versions():
1275
result.add(prefix + (suffix,))
1279
class _PlanMergeVersionedFile(VersionedFiles):
1280
"""A VersionedFile for uncommitted and committed texts.
1282
It is intended to allow merges to be planned with working tree texts.
1283
It implements only the small part of the VersionedFiles interface used by
1284
PlanMerge. It falls back to multiple versionedfiles for data not stored in
1285
_PlanMergeVersionedFile itself.
1287
:ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1288
queried for missing texts.
1291
def __init__(self, file_id):
1292
"""Create a _PlanMergeVersionedFile.
1294
:param file_id: Used with _PlanMerge code which is not yet fully
1295
tuple-keyspace aware.
1297
self._file_id = file_id
1298
# fallback locations
1299
self.fallback_versionedfiles = []
1300
# Parents for locally held keys.
1302
# line data for locally held keys.
1304
# key lookup providers
1305
self._providers = [DictParentsProvider(self._parents)]
1307
def plan_merge(self, ver_a, ver_b, base=None):
1308
"""See VersionedFile.plan_merge"""
1309
from bzrlib.merge import _PlanMerge
1311
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1312
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1313
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1314
return _PlanMerge._subtract_plans(old_plan, new_plan)
1316
def plan_lca_merge(self, ver_a, ver_b, base=None):
1317
from bzrlib.merge import _PlanLCAMerge
1319
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1322
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1323
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1325
def add_lines(self, key, parents, lines):
1326
"""See VersionedFiles.add_lines
1328
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1329
are permitted. Only reserved ids are permitted.
1331
if type(key) is not tuple:
1332
raise TypeError(key)
1333
if not revision.is_reserved_id(key[-1]):
1334
raise ValueError('Only reserved ids may be used')
1336
raise ValueError('Parents may not be None')
1338
raise ValueError('Lines may not be None')
1339
self._parents[key] = tuple(parents)
1340
self._lines[key] = lines
1342
def get_record_stream(self, keys, ordering, include_delta_closure):
1345
if key in self._lines:
1346
lines = self._lines[key]
1347
parents = self._parents[key]
1349
yield ChunkedContentFactory(key, parents, None, lines)
1350
for versionedfile in self.fallback_versionedfiles:
1351
for record in versionedfile.get_record_stream(
1352
pending, 'unordered', True):
1353
if record.storage_kind == 'absent':
1356
pending.remove(record.key)
1360
# report absent entries
1362
yield AbsentContentFactory(key)
1364
def get_parent_map(self, keys):
1365
"""See VersionedFiles.get_parent_map"""
1366
# We create a new provider because a fallback may have been added.
1367
# If we make fallbacks private we can update a stack list and avoid
1368
# object creation thrashing.
1371
if revision.NULL_REVISION in keys:
1372
keys.remove(revision.NULL_REVISION)
1373
result[revision.NULL_REVISION] = ()
1374
self._providers = self._providers[:1] + self.fallback_versionedfiles
1376
StackedParentsProvider(self._providers).get_parent_map(keys))
1377
for key, parents in result.iteritems():
1379
result[key] = (revision.NULL_REVISION,)
1383
class PlanWeaveMerge(TextMerge):
1384
"""Weave merge that takes a plan as its input.
1386
This exists so that VersionedFile.plan_merge is implementable.
1387
Most callers will want to use WeaveMerge instead.
1390
def __init__(self, plan, a_marker=TextMerge.A_MARKER,
1391
b_marker=TextMerge.B_MARKER):
1392
TextMerge.__init__(self, a_marker, b_marker)
1395
def _merge_struct(self):
1400
def outstanding_struct():
1401
if not lines_a and not lines_b:
1403
elif ch_a and not ch_b:
1406
elif ch_b and not ch_a:
1408
elif lines_a == lines_b:
1411
yield (lines_a, lines_b)
1413
# We previously considered either 'unchanged' or 'killed-both' lines
1414
# to be possible places to resynchronize. However, assuming agreement
1415
# on killed-both lines may be too aggressive. -- mbp 20060324
1416
for state, line in self.plan:
1417
if state == 'unchanged':
1418
# resync and flush queued conflicts changes if any
1419
for struct in outstanding_struct():
1425
if state == 'unchanged':
1428
elif state == 'killed-a':
1430
lines_b.append(line)
1431
elif state == 'killed-b':
1433
lines_a.append(line)
1434
elif state == 'new-a':
1436
lines_a.append(line)
1437
elif state == 'new-b':
1439
lines_b.append(line)
1440
elif state == 'conflicted-a':
1442
lines_a.append(line)
1443
elif state == 'conflicted-b':
1445
lines_b.append(line)
1446
elif state == 'killed-both':
1447
# This counts as a change, even though there is no associated
1451
if state not in ('irrelevant', 'ghost-a', 'ghost-b',
1453
raise AssertionError(state)
1454
for struct in outstanding_struct():
1458
class WeaveMerge(PlanWeaveMerge):
1459
"""Weave merge that takes a VersionedFile and two versions as its input."""
1461
def __init__(self, versionedfile, ver_a, ver_b,
1462
a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1463
plan = versionedfile.plan_merge(ver_a, ver_b)
1464
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1467
class VirtualVersionedFiles(VersionedFiles):
1468
"""Dummy implementation for VersionedFiles that uses other functions for
1469
obtaining fulltexts and parent maps.
1471
This is always on the bottom of the stack and uses string keys
1472
(rather than tuples) internally.
1475
def __init__(self, get_parent_map, get_lines):
1476
"""Create a VirtualVersionedFiles.
1478
:param get_parent_map: Same signature as Repository.get_parent_map.
1479
:param get_lines: Should return lines for specified key or None if
1482
super(VirtualVersionedFiles, self).__init__()
1483
self._get_parent_map = get_parent_map
1484
self._get_lines = get_lines
1486
def check(self, progressbar=None):
1487
"""See VersionedFiles.check.
1489
:note: Always returns True for VirtualVersionedFiles.
1493
def add_mpdiffs(self, records):
1494
"""See VersionedFiles.mpdiffs.
1496
:note: Not implemented for VirtualVersionedFiles.
1498
raise NotImplementedError(self.add_mpdiffs)
1500
def get_parent_map(self, keys):
1501
"""See VersionedFiles.get_parent_map."""
1502
return dict([((k,), tuple([(p,) for p in v]))
1503
for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
1505
def get_sha1s(self, keys):
1506
"""See VersionedFiles.get_sha1s."""
1509
lines = self._get_lines(k)
1510
if lines is not None:
1511
if not isinstance(lines, list):
1512
raise AssertionError
1513
ret[(k,)] = osutils.sha_strings(lines)
1516
def get_record_stream(self, keys, ordering, include_delta_closure):
1517
"""See VersionedFiles.get_record_stream."""
1518
for (k,) in list(keys):
1519
lines = self._get_lines(k)
1520
if lines is not None:
1521
if not isinstance(lines, list):
1522
raise AssertionError
1523
yield ChunkedContentFactory((k,), None,
1524
sha1=osutils.sha_strings(lines),
1527
yield AbsentContentFactory((k,))
1529
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1530
"""See VersionedFile.iter_lines_added_or_present_in_versions()."""
1531
for i, (key,) in enumerate(keys):
1533
pb.update("Finding changed lines", i, len(keys))
1534
for l in self._get_lines(key):
1538
def network_bytes_to_kind_and_offset(network_bytes):
1539
"""Strip of a record kind from the front of network_bytes.
1541
:param network_bytes: The bytes of a record.
1542
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1544
line_end = network_bytes.find('\n')
1545
storage_kind = network_bytes[:line_end]
1546
return storage_kind, line_end + 1
1549
class NetworkRecordStream(object):
1550
"""A record_stream which reconstitures a serialised stream."""
1552
def __init__(self, bytes_iterator):
1553
"""Create a NetworkRecordStream.
1555
:param bytes_iterator: An iterator of bytes. Each item in this
1556
iterator should have been obtained from a record_streams'
1557
record.get_bytes_as(record.storage_kind) call.
1559
self._bytes_iterator = bytes_iterator
1560
self._kind_factory = {'knit-ft-gz':knit.knit_network_to_record,
1561
'knit-delta-gz':knit.knit_network_to_record,
1562
'knit-annotated-ft-gz':knit.knit_network_to_record,
1563
'knit-annotated-delta-gz':knit.knit_network_to_record,
1564
'knit-delta-closure':knit.knit_delta_closure_to_records,
1565
'fulltext':fulltext_network_to_record,
1566
'groupcompress-block':groupcompress.network_block_to_records,
1572
:return: An iterator as per VersionedFiles.get_record_stream().
1574
for bytes in self._bytes_iterator:
1575
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1576
for record in self._kind_factory[storage_kind](
1577
storage_kind, bytes, line_end):
1581
def fulltext_network_to_record(kind, bytes, line_end):
1582
"""Convert a network fulltext record to record."""
1583
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1584
record_meta = bytes[line_end+4:line_end+4+meta_len]
1585
key, parents = bencode.bdecode_as_tuple(record_meta)
1586
if parents == 'nil':
1588
fulltext = bytes[line_end+4+meta_len:]
1589
return [FulltextContentFactory(key, parents, None, fulltext)]
1592
def _length_prefix(bytes):
1593
return struct.pack('!L', len(bytes))
1596
def record_to_fulltext_bytes(record):
1597
if record.parents is None:
1600
parents = record.parents
1601
record_meta = bencode.bencode((record.key, parents))
1602
record_content = record.get_bytes_as('fulltext')
1603
return "fulltext\n%s%s%s" % (
1604
_length_prefix(record_meta), record_meta, record_content)
1607
def sort_groupcompress(parent_map):
1608
"""Sort and group the keys in parent_map into groupcompress order.
1610
groupcompress is defined (currently) as reverse-topological order, grouped
1613
:return: A sorted-list of keys
1615
# gc-optimal ordering is approximately reverse topological,
1616
# properly grouped by file-id.
1618
for item in parent_map.iteritems():
1620
if isinstance(key, str) or len(key) == 1:
1625
per_prefix_map[prefix].append(item)
1627
per_prefix_map[prefix] = [item]
1630
for prefix in sorted(per_prefix_map):
1631
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
added
removed
bzrlib/versionedfile.py
