1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20
"""Versioned text file storage api."""
23
from cStringIO import StringIO
26
from zlib import adler32
28
from bzrlib.lazy_import import lazy_import
29
lazy_import(globals(), """
46
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
47
from bzrlib.transport.memory import MemoryTransport
49
from bzrlib.inter import InterObject
50
from bzrlib.registry import Registry
51
from bzrlib.symbol_versioning import *
52
from bzrlib.textmerge import TextMerge
53
from bzrlib import bencode
56
adapter_registry = Registry()
57
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
58
'DeltaPlainToFullText')
59
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
62
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
63
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
64
'bzrlib.knit', 'DeltaAnnotatedToFullText')
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
66
'bzrlib.knit', 'FTAnnotatedToUnannotated')
67
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
68
'bzrlib.knit', 'FTAnnotatedToFullText')
69
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
70
# 'bzrlib.knit', 'FTAnnotatedToChunked')
73
class ContentFactory(object):
74
"""Abstract interface for insertion and retrieval from a VersionedFile.
76
:ivar sha1: None, or the sha1 of the content fulltext.
77
:ivar storage_kind: The native storage kind of this factory. One of
78
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
79
'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
80
'knit-annotated-delta-gz', 'knit-ft-gz', 'knit-delta-gz'.
81
:ivar key: The key of this content. Each key is a tuple with a single
83
:ivar parents: A tuple of parent keys for self.key. If the object has
84
no parent information, None (as opposed to () for an empty list of
89
"""Create a ContentFactory."""
91
self.storage_kind = None
96
class ChunkedContentFactory(ContentFactory):
97
"""Static data content factory.
99
This takes a 'chunked' list of strings. The only requirement on 'chunked' is
100
that ''.join(lines) becomes a valid fulltext. A tuple of a single string
101
satisfies this, as does a list of lines.
103
:ivar sha1: None, or the sha1 of the content fulltext.
104
:ivar storage_kind: The native storage kind of this factory. Always
106
:ivar key: The key of this content. Each key is a tuple with a single
108
:ivar parents: A tuple of parent keys for self.key. If the object has
109
no parent information, None (as opposed to () for an empty list of
113
def __init__(self, key, parents, sha1, chunks):
114
"""Create a ContentFactory."""
116
self.storage_kind = 'chunked'
118
self.parents = parents
119
self._chunks = chunks
121
def get_bytes_as(self, storage_kind):
122
if storage_kind == 'chunked':
124
elif storage_kind == 'fulltext':
125
return ''.join(self._chunks)
126
raise errors.UnavailableRepresentation(self.key, storage_kind,
130
class FulltextContentFactory(ContentFactory):
131
"""Static data content factory.
133
This takes a fulltext when created and just returns that during
134
get_bytes_as('fulltext').
136
:ivar sha1: None, or the sha1 of the content fulltext.
137
:ivar storage_kind: The native storage kind of this factory. Always
139
:ivar key: The key of this content. Each key is a tuple with a single
141
:ivar parents: A tuple of parent keys for self.key. If the object has
142
no parent information, None (as opposed to () for an empty list of
146
def __init__(self, key, parents, sha1, text):
147
"""Create a ContentFactory."""
149
self.storage_kind = 'fulltext'
151
self.parents = parents
154
def get_bytes_as(self, storage_kind):
155
if storage_kind == self.storage_kind:
157
elif storage_kind == 'chunked':
159
raise errors.UnavailableRepresentation(self.key, storage_kind,
163
class InventoryDeltaContentFactory(ContentFactory):
165
def __init__(self, key, parents, sha1, delta, basis_id, format_flags,
168
self.storage_kind = 'inventory-delta'
170
self.parents = parents
172
self._basis_id = basis_id
173
self._format_flags = format_flags
176
def get_bytes_as(self, storage_kind):
177
if storage_kind == self.storage_kind:
178
return self._basis_id, self.key, self._delta, self._format_flags
179
elif storage_kind == 'inventory-delta-bytes':
180
serializer = inventory_delta.InventoryDeltaSerializer()
181
serializer.require_flags(*self._format_flags)
182
return ''.join(serializer.delta_to_lines(
183
self._basis_id, self.key, self._delta))
184
elif storage_kind == 'inventory-delta-bytes-from-null':
185
if self._repo is None:
186
raise errors.UnavailableRepresentation(self.key, storage_kind,
188
null_inv = inventory.Inventory(None)
189
my_inv = self._repo.get_inventory(self.key) # XXX: key[0] ???
190
delta = my_inv._make_delta(null_inv)
191
serializer.require_flags(*self._format_flags)
192
return serializer.delta_to_lines(
193
revision.NULL_REVISION, self.key, delta)
194
raise errors.UnavailableRepresentation(self.key, storage_kind,
198
class AbsentContentFactory(ContentFactory):
199
"""A placeholder content factory for unavailable texts.
202
:ivar storage_kind: 'absent'.
203
:ivar key: The key of this content. Each key is a tuple with a single
208
def __init__(self, key):
209
"""Create a ContentFactory."""
211
self.storage_kind = 'absent'
216
class AdapterFactory(ContentFactory):
217
"""A content factory to adapt between key prefix's."""
219
def __init__(self, key, parents, adapted):
220
"""Create an adapter factory instance."""
222
self.parents = parents
223
self._adapted = adapted
225
def __getattr__(self, attr):
226
"""Return a member from the adapted object."""
227
if attr in ('key', 'parents'):
228
return self.__dict__[attr]
230
return getattr(self._adapted, attr)
233
def filter_absent(record_stream):
234
"""Adapt a record stream to remove absent records."""
235
for record in record_stream:
236
if record.storage_kind != 'absent':
240
class VersionedFile(object):
241
"""Versioned text file storage.
243
A versioned file manages versions of line-based text files,
244
keeping track of the originating version for each line.
246
To clients the "lines" of the file are represented as a list of
247
strings. These strings will typically have terminal newline
248
characters, but this is not required. In particular files commonly
249
do not have a newline at the end of the file.
251
Texts are identified by a version-id string.
255
def check_not_reserved_id(version_id):
256
revision.check_not_reserved_id(version_id)
258
def copy_to(self, name, transport):
259
"""Copy this versioned file to name on transport."""
260
raise NotImplementedError(self.copy_to)
262
def get_record_stream(self, versions, ordering, include_delta_closure):
263
"""Get a stream of records for versions.
265
:param versions: The versions to include. Each version is a tuple
267
:param ordering: Either 'unordered' or 'topological'. A topologically
268
sorted stream has compression parents strictly before their
270
:param include_delta_closure: If True then the closure across any
271
compression parents will be included (in the data content of the
272
stream, not in the emitted records). This guarantees that
273
'fulltext' can be used successfully on every record.
274
:return: An iterator of ContentFactory objects, each of which is only
275
valid until the iterator is advanced.
277
raise NotImplementedError(self.get_record_stream)
279
def has_version(self, version_id):
280
"""Returns whether version is present."""
281
raise NotImplementedError(self.has_version)
283
def insert_record_stream(self, stream):
284
"""Insert a record stream into this versioned file.
286
:param stream: A stream of records to insert.
288
:seealso VersionedFile.get_record_stream:
290
raise NotImplementedError
292
def add_lines(self, version_id, parents, lines, parent_texts=None,
293
left_matching_blocks=None, nostore_sha=None, random_id=False,
295
"""Add a single text on top of the versioned file.
297
Must raise RevisionAlreadyPresent if the new version is
298
already present in file history.
300
Must raise RevisionNotPresent if any of the given parents are
301
not present in file history.
303
:param lines: A list of lines. Each line must be a bytestring. And all
304
of them except the last must be terminated with \n and contain no
305
other \n's. The last line may either contain no \n's or a single
306
terminated \n. If the lines list does meet this constraint the add
307
routine may error or may succeed - but you will be unable to read
308
the data back accurately. (Checking the lines have been split
309
correctly is expensive and extremely unlikely to catch bugs so it
310
is not done at runtime unless check_content is True.)
311
:param parent_texts: An optional dictionary containing the opaque
312
representations of some or all of the parents of version_id to
313
allow delta optimisations. VERY IMPORTANT: the texts must be those
314
returned by add_lines or data corruption can be caused.
315
:param left_matching_blocks: a hint about which areas are common
316
between the text and its left-hand-parent. The format is
317
the SequenceMatcher.get_matching_blocks format.
318
:param nostore_sha: Raise ExistingContent and do not add the lines to
319
the versioned file if the digest of the lines matches this.
320
:param random_id: If True a random id has been selected rather than
321
an id determined by some deterministic process such as a converter
322
from a foreign VCS. When True the backend may choose not to check
323
for uniqueness of the resulting key within the versioned file, so
324
this should only be done when the result is expected to be unique
326
:param check_content: If True, the lines supplied are verified to be
327
bytestrings that are correctly formed lines.
328
:return: The text sha1, the number of bytes in the text, and an opaque
329
representation of the inserted version which can be provided
330
back to future add_lines calls in the parent_texts dictionary.
332
self._check_write_ok()
333
return self._add_lines(version_id, parents, lines, parent_texts,
334
left_matching_blocks, nostore_sha, random_id, check_content)
336
def _add_lines(self, version_id, parents, lines, parent_texts,
337
left_matching_blocks, nostore_sha, random_id, check_content):
338
"""Helper to do the class specific add_lines."""
339
raise NotImplementedError(self.add_lines)
341
def add_lines_with_ghosts(self, version_id, parents, lines,
342
parent_texts=None, nostore_sha=None, random_id=False,
343
check_content=True, left_matching_blocks=None):
344
"""Add lines to the versioned file, allowing ghosts to be present.
346
This takes the same parameters as add_lines and returns the same.
348
self._check_write_ok()
349
return self._add_lines_with_ghosts(version_id, parents, lines,
350
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
352
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
353
nostore_sha, random_id, check_content, left_matching_blocks):
354
"""Helper to do class specific add_lines_with_ghosts."""
355
raise NotImplementedError(self.add_lines_with_ghosts)
357
def check(self, progress_bar=None):
358
"""Check the versioned file for integrity."""
359
raise NotImplementedError(self.check)
361
def _check_lines_not_unicode(self, lines):
362
"""Check that lines being added to a versioned file are not unicode."""
364
if line.__class__ is not str:
365
raise errors.BzrBadParameterUnicode("lines")
367
def _check_lines_are_lines(self, lines):
368
"""Check that the lines really are full lines without inline EOL."""
370
if '\n' in line[:-1]:
371
raise errors.BzrBadParameterContainsNewline("lines")
373
def get_format_signature(self):
374
"""Get a text description of the data encoding in this file.
378
raise NotImplementedError(self.get_format_signature)
380
def make_mpdiffs(self, version_ids):
381
"""Create multiparent diffs for specified versions."""
382
knit_versions = set()
383
knit_versions.update(version_ids)
384
parent_map = self.get_parent_map(version_ids)
385
for version_id in version_ids:
387
knit_versions.update(parent_map[version_id])
389
raise errors.RevisionNotPresent(version_id, self)
390
# We need to filter out ghosts, because we can't diff against them.
391
knit_versions = set(self.get_parent_map(knit_versions).keys())
392
lines = dict(zip(knit_versions,
393
self._get_lf_split_line_list(knit_versions)))
395
for version_id in version_ids:
396
target = lines[version_id]
398
parents = [lines[p] for p in parent_map[version_id] if p in
401
# I don't know how this could ever trigger.
402
# parent_map[version_id] was already triggered in the previous
403
# for loop, and lines[p] has the 'if p in knit_versions' check,
404
# so we again won't have a KeyError.
405
raise errors.RevisionNotPresent(version_id, self)
407
left_parent_blocks = self._extract_blocks(version_id,
410
left_parent_blocks = None
411
diffs.append(multiparent.MultiParent.from_lines(target, parents,
415
def _extract_blocks(self, version_id, source, target):
418
def add_mpdiffs(self, records):
419
"""Add mpdiffs to this VersionedFile.
421
Records should be iterables of version, parents, expected_sha1,
422
mpdiff. mpdiff should be a MultiParent instance.
424
# Does this need to call self._check_write_ok()? (IanC 20070919)
426
mpvf = multiparent.MultiMemoryVersionedFile()
428
for version, parent_ids, expected_sha1, mpdiff in records:
429
versions.append(version)
430
mpvf.add_diff(mpdiff, version, parent_ids)
431
needed_parents = set()
432
for version, parent_ids, expected_sha1, mpdiff in records:
433
needed_parents.update(p for p in parent_ids
434
if not mpvf.has_version(p))
435
present_parents = set(self.get_parent_map(needed_parents).keys())
436
for parent_id, lines in zip(present_parents,
437
self._get_lf_split_line_list(present_parents)):
438
mpvf.add_version(lines, parent_id, [])
439
for (version, parent_ids, expected_sha1, mpdiff), lines in\
440
zip(records, mpvf.get_line_list(versions)):
441
if len(parent_ids) == 1:
442
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
443
mpvf.get_diff(parent_ids[0]).num_lines()))
445
left_matching_blocks = None
447
_, _, version_text = self.add_lines_with_ghosts(version,
448
parent_ids, lines, vf_parents,
449
left_matching_blocks=left_matching_blocks)
450
except NotImplementedError:
451
# The vf can't handle ghosts, so add lines normally, which will
452
# (reasonably) fail if there are ghosts in the data.
453
_, _, version_text = self.add_lines(version,
454
parent_ids, lines, vf_parents,
455
left_matching_blocks=left_matching_blocks)
456
vf_parents[version] = version_text
457
sha1s = self.get_sha1s(versions)
458
for version, parent_ids, expected_sha1, mpdiff in records:
459
if expected_sha1 != sha1s[version]:
460
raise errors.VersionedFileInvalidChecksum(version)
462
def get_text(self, version_id):
463
"""Return version contents as a text string.
465
Raises RevisionNotPresent if version is not present in
468
return ''.join(self.get_lines(version_id))
469
get_string = get_text
471
def get_texts(self, version_ids):
472
"""Return the texts of listed versions as a list of strings.
474
Raises RevisionNotPresent if version is not present in
477
return [''.join(self.get_lines(v)) for v in version_ids]
479
def get_lines(self, version_id):
480
"""Return version contents as a sequence of lines.
482
Raises RevisionNotPresent if version is not present in
485
raise NotImplementedError(self.get_lines)
487
def _get_lf_split_line_list(self, version_ids):
488
return [StringIO(t).readlines() for t in self.get_texts(version_ids)]
490
def get_ancestry(self, version_ids, topo_sorted=True):
491
"""Return a list of all ancestors of given version(s). This
492
will not include the null revision.
494
This list will not be topologically sorted if topo_sorted=False is
497
Must raise RevisionNotPresent if any of the given versions are
498
not present in file history."""
499
if isinstance(version_ids, basestring):
500
version_ids = [version_ids]
501
raise NotImplementedError(self.get_ancestry)
503
def get_ancestry_with_ghosts(self, version_ids):
504
"""Return a list of all ancestors of given version(s). This
505
will not include the null revision.
507
Must raise RevisionNotPresent if any of the given versions are
508
not present in file history.
510
Ghosts that are known about will be included in ancestry list,
511
but are not explicitly marked.
513
raise NotImplementedError(self.get_ancestry_with_ghosts)
515
def get_parent_map(self, version_ids):
516
"""Get a map of the parents of version_ids.
518
:param version_ids: The version ids to look up parents for.
519
:return: A mapping from version id to parents.
521
raise NotImplementedError(self.get_parent_map)
523
def get_parents_with_ghosts(self, version_id):
524
"""Return version names for parents of version_id.
526
Will raise RevisionNotPresent if version_id is not present
529
Ghosts that are known about will be included in the parent list,
530
but are not explicitly marked.
533
return list(self.get_parent_map([version_id])[version_id])
535
raise errors.RevisionNotPresent(version_id, self)
537
def annotate(self, version_id):
538
"""Return a list of (version-id, line) tuples for version_id.
540
:raise RevisionNotPresent: If the given version is
541
not present in file history.
543
raise NotImplementedError(self.annotate)
545
def iter_lines_added_or_present_in_versions(self, version_ids=None,
547
"""Iterate over the lines in the versioned file from version_ids.
549
This may return lines from other versions. Each item the returned
550
iterator yields is a tuple of a line and a text version that that line
551
is present in (not introduced in).
553
Ordering of results is in whatever order is most suitable for the
554
underlying storage format.
556
If a progress bar is supplied, it may be used to indicate progress.
557
The caller is responsible for cleaning up progress bars (because this
560
NOTES: Lines are normalised: they will all have \n terminators.
561
Lines are returned in arbitrary order.
563
:return: An iterator over (line, version_id).
565
raise NotImplementedError(self.iter_lines_added_or_present_in_versions)
567
def plan_merge(self, ver_a, ver_b):
568
"""Return pseudo-annotation indicating how the two versions merge.
570
This is computed between versions a and b and their common
573
Weave lines present in none of them are skipped entirely.
576
killed-base Dead in base revision
577
killed-both Killed in each revision
580
unchanged Alive in both a and b (possibly created in both)
583
ghost-a Killed in a, unborn in b
584
ghost-b Killed in b, unborn in a
585
irrelevant Not in either revision
587
raise NotImplementedError(VersionedFile.plan_merge)
589
def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
590
b_marker=TextMerge.B_MARKER):
591
return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
594
class RecordingVersionedFilesDecorator(object):
595
"""A minimal versioned files that records calls made on it.
597
Only enough methods have been added to support tests using it to date.
599
:ivar calls: A list of the calls made; can be reset at any time by
603
def __init__(self, backing_vf):
604
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
606
:param backing_vf: The versioned file to answer all methods.
608
self._backing_vf = backing_vf
611
def add_lines(self, key, parents, lines, parent_texts=None,
612
left_matching_blocks=None, nostore_sha=None, random_id=False,
614
self.calls.append(("add_lines", key, parents, lines, parent_texts,
615
left_matching_blocks, nostore_sha, random_id, check_content))
616
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
617
left_matching_blocks, nostore_sha, random_id, check_content)
620
self._backing_vf.check()
622
def get_parent_map(self, keys):
623
self.calls.append(("get_parent_map", copy(keys)))
624
return self._backing_vf.get_parent_map(keys)
626
def get_record_stream(self, keys, sort_order, include_delta_closure):
627
self.calls.append(("get_record_stream", list(keys), sort_order,
628
include_delta_closure))
629
return self._backing_vf.get_record_stream(keys, sort_order,
630
include_delta_closure)
632
def get_sha1s(self, keys):
633
self.calls.append(("get_sha1s", copy(keys)))
634
return self._backing_vf.get_sha1s(keys)
636
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
637
self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
638
return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
641
self.calls.append(("keys",))
642
return self._backing_vf.keys()
645
class OrderingVersionedFilesDecorator(RecordingVersionedFilesDecorator):
646
"""A VF that records calls, and returns keys in specific order.
648
:ivar calls: A list of the calls made; can be reset at any time by
652
def __init__(self, backing_vf, key_priority):
653
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
655
:param backing_vf: The versioned file to answer all methods.
656
:param key_priority: A dictionary defining what order keys should be
657
returned from an 'unordered' get_record_stream request.
658
Keys with lower priority are returned first, keys not present in
659
the map get an implicit priority of 0, and are returned in
660
lexicographical order.
662
RecordingVersionedFilesDecorator.__init__(self, backing_vf)
663
self._key_priority = key_priority
665
def get_record_stream(self, keys, sort_order, include_delta_closure):
666
self.calls.append(("get_record_stream", list(keys), sort_order,
667
include_delta_closure))
668
if sort_order == 'unordered':
670
return (self._key_priority.get(key, 0), key)
671
# Use a defined order by asking for the keys one-by-one from the
673
for key in sorted(keys, key=sort_key):
674
for record in self._backing_vf.get_record_stream([key],
675
'unordered', include_delta_closure):
678
for record in self._backing_vf.get_record_stream(keys, sort_order,
679
include_delta_closure):
683
class KeyMapper(object):
684
"""KeyMappers map between keys and underlying partitioned storage."""
687
"""Map key to an underlying storage identifier.
689
:param key: A key tuple e.g. ('file-id', 'revision-id').
690
:return: An underlying storage identifier, specific to the partitioning
693
raise NotImplementedError(self.map)
695
def unmap(self, partition_id):
696
"""Map a partitioned storage id back to a key prefix.
698
:param partition_id: The underlying partition id.
699
:return: As much of a key (or prefix) as is derivable from the partition
702
raise NotImplementedError(self.unmap)
705
class ConstantMapper(KeyMapper):
706
"""A key mapper that maps to a constant result."""
708
def __init__(self, result):
709
"""Create a ConstantMapper which will return result for all maps."""
710
self._result = result
713
"""See KeyMapper.map()."""
717
class URLEscapeMapper(KeyMapper):
718
"""Base class for use with transport backed storage.
720
This provides a map and unmap wrapper that respectively url escape and
721
unescape their outputs and inputs.
725
"""See KeyMapper.map()."""
726
return urllib.quote(self._map(key))
728
def unmap(self, partition_id):
729
"""See KeyMapper.unmap()."""
730
return self._unmap(urllib.unquote(partition_id))
733
class PrefixMapper(URLEscapeMapper):
734
"""A key mapper that extracts the first component of a key.
736
This mapper is for use with a transport based backend.
740
"""See KeyMapper.map()."""
743
def _unmap(self, partition_id):
744
"""See KeyMapper.unmap()."""
745
return (partition_id,)
748
class HashPrefixMapper(URLEscapeMapper):
749
"""A key mapper that combines the first component of a key with a hash.
751
This mapper is for use with a transport based backend.
755
"""See KeyMapper.map()."""
756
prefix = self._escape(key[0])
757
return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
759
def _escape(self, prefix):
760
"""No escaping needed here."""
763
def _unmap(self, partition_id):
764
"""See KeyMapper.unmap()."""
765
return (self._unescape(osutils.basename(partition_id)),)
767
def _unescape(self, basename):
768
"""No unescaping needed for HashPrefixMapper."""
772
class HashEscapedPrefixMapper(HashPrefixMapper):
773
"""Combines the escaped first component of a key with a hash.
775
This mapper is for use with a transport based backend.
778
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
780
def _escape(self, prefix):
781
"""Turn a key element into a filesystem safe string.
783
This is similar to a plain urllib.quote, except
784
it uses specific safe characters, so that it doesn't
785
have to translate a lot of valid file ids.
787
# @ does not get escaped. This is because it is a valid
788
# filesystem character we use all the time, and it looks
789
# a lot better than seeing %40 all the time.
790
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
794
def _unescape(self, basename):
795
"""Escaped names are easily unescaped by urlutils."""
796
return urllib.unquote(basename)
799
def make_versioned_files_factory(versioned_file_factory, mapper):
800
"""Create a ThunkedVersionedFiles factory.
802
This will create a callable which when called creates a
803
ThunkedVersionedFiles on a transport, using mapper to access individual
804
versioned files, and versioned_file_factory to create each individual file.
806
def factory(transport):
807
return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
812
class VersionedFiles(object):
813
"""Storage for many versioned files.
815
This object allows a single keyspace for accessing the history graph and
816
contents of named bytestrings.
818
Currently no implementation allows the graph of different key prefixes to
819
intersect, but the API does allow such implementations in the future.
821
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
822
may have a different length key-size, but that size will be constant for
823
all texts added to or retrieved from it. For instance, bzrlib uses
824
instances with a key-size of 2 for storing user files in a repository, with
825
the first element the fileid, and the second the version of that file.
827
The use of tuples allows a single code base to support several different
828
uses with only the mapping logic changing from instance to instance.
831
def add_lines(self, key, parents, lines, parent_texts=None,
832
left_matching_blocks=None, nostore_sha=None, random_id=False,
834
"""Add a text to the store.
836
:param key: The key tuple of the text to add. If the last element is
837
None, a CHK string will be generated during the addition.
838
:param parents: The parents key tuples of the text to add.
839
:param lines: A list of lines. Each line must be a bytestring. And all
840
of them except the last must be terminated with \n and contain no
841
other \n's. The last line may either contain no \n's or a single
842
terminating \n. If the lines list does meet this constraint the add
843
routine may error or may succeed - but you will be unable to read
844
the data back accurately. (Checking the lines have been split
845
correctly is expensive and extremely unlikely to catch bugs so it
846
is not done at runtime unless check_content is True.)
847
:param parent_texts: An optional dictionary containing the opaque
848
representations of some or all of the parents of version_id to
849
allow delta optimisations. VERY IMPORTANT: the texts must be those
850
returned by add_lines or data corruption can be caused.
851
:param left_matching_blocks: a hint about which areas are common
852
between the text and its left-hand-parent. The format is
853
the SequenceMatcher.get_matching_blocks format.
854
:param nostore_sha: Raise ExistingContent and do not add the lines to
855
the versioned file if the digest of the lines matches this.
856
:param random_id: If True a random id has been selected rather than
857
an id determined by some deterministic process such as a converter
858
from a foreign VCS. When True the backend may choose not to check
859
for uniqueness of the resulting key within the versioned file, so
860
this should only be done when the result is expected to be unique
862
:param check_content: If True, the lines supplied are verified to be
863
bytestrings that are correctly formed lines.
864
:return: The text sha1, the number of bytes in the text, and an opaque
865
representation of the inserted version which can be provided
866
back to future add_lines calls in the parent_texts dictionary.
868
raise NotImplementedError(self.add_lines)
870
def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
871
"""Add a text to the store.
873
This is a private function for use by CommitBuilder.
875
:param key: The key tuple of the text to add. If the last element is
876
None, a CHK string will be generated during the addition.
877
:param parents: The parents key tuples of the text to add.
878
:param text: A string containing the text to be committed.
879
:param nostore_sha: Raise ExistingContent and do not add the lines to
880
the versioned file if the digest of the lines matches this.
881
:param random_id: If True a random id has been selected rather than
882
an id determined by some deterministic process such as a converter
883
from a foreign VCS. When True the backend may choose not to check
884
for uniqueness of the resulting key within the versioned file, so
885
this should only be done when the result is expected to be unique
887
:param check_content: If True, the lines supplied are verified to be
888
bytestrings that are correctly formed lines.
889
:return: The text sha1, the number of bytes in the text, and an opaque
890
representation of the inserted version which can be provided
891
back to future _add_text calls in the parent_texts dictionary.
893
# The default implementation just thunks over to .add_lines(),
894
# inefficient, but it works.
895
return self.add_lines(key, parents, osutils.split_lines(text),
896
nostore_sha=nostore_sha,
900
def add_mpdiffs(self, records):
901
"""Add mpdiffs to this VersionedFile.
903
Records should be iterables of version, parents, expected_sha1,
904
mpdiff. mpdiff should be a MultiParent instance.
907
mpvf = multiparent.MultiMemoryVersionedFile()
909
for version, parent_ids, expected_sha1, mpdiff in records:
910
versions.append(version)
911
mpvf.add_diff(mpdiff, version, parent_ids)
912
needed_parents = set()
913
for version, parent_ids, expected_sha1, mpdiff in records:
914
needed_parents.update(p for p in parent_ids
915
if not mpvf.has_version(p))
916
# It seems likely that adding all the present parents as fulltexts can
917
# easily exhaust memory.
918
chunks_to_lines = osutils.chunks_to_lines
919
for record in self.get_record_stream(needed_parents, 'unordered',
921
if record.storage_kind == 'absent':
923
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
925
for (key, parent_keys, expected_sha1, mpdiff), lines in\
926
zip(records, mpvf.get_line_list(versions)):
927
if len(parent_keys) == 1:
928
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
929
mpvf.get_diff(parent_keys[0]).num_lines()))
931
left_matching_blocks = None
932
version_sha1, _, version_text = self.add_lines(key,
933
parent_keys, lines, vf_parents,
934
left_matching_blocks=left_matching_blocks)
935
if version_sha1 != expected_sha1:
936
raise errors.VersionedFileInvalidChecksum(version)
937
vf_parents[key] = version_text
939
def annotate(self, key):
940
"""Return a list of (version-key, line) tuples for the text of key.
942
:raise RevisionNotPresent: If the key is not present.
944
raise NotImplementedError(self.annotate)
946
def check(self, progress_bar=None):
947
"""Check this object for integrity."""
948
raise NotImplementedError(self.check)
951
def check_not_reserved_id(version_id):
952
revision.check_not_reserved_id(version_id)
954
def _check_lines_not_unicode(self, lines):
955
"""Check that lines being added to a versioned file are not unicode."""
957
if line.__class__ is not str:
958
raise errors.BzrBadParameterUnicode("lines")
960
def _check_lines_are_lines(self, lines):
961
"""Check that the lines really are full lines without inline EOL."""
963
if '\n' in line[:-1]:
964
raise errors.BzrBadParameterContainsNewline("lines")
966
def get_parent_map(self, keys):
967
"""Get a map of the parents of keys.
969
:param keys: The keys to look up parents for.
970
:return: A mapping from keys to parents. Absent keys are absent from
973
raise NotImplementedError(self.get_parent_map)
975
def get_record_stream(self, keys, ordering, include_delta_closure):
976
"""Get a stream of records for keys.
978
:param keys: The keys to include.
979
:param ordering: Either 'unordered' or 'topological'. A topologically
980
sorted stream has compression parents strictly before their
982
:param include_delta_closure: If True then the closure across any
983
compression parents will be included (in the opaque data).
984
:return: An iterator of ContentFactory objects, each of which is only
985
valid until the iterator is advanced.
987
raise NotImplementedError(self.get_record_stream)
989
def get_sha1s(self, keys):
990
"""Get the sha1's of the texts for the given keys.
992
:param keys: The names of the keys to lookup
993
:return: a dict from key to sha1 digest. Keys of texts which are not
994
present in the store are not present in the returned
997
raise NotImplementedError(self.get_sha1s)
999
has_key = index._has_key_from_parent_map
1001
def get_missing_compression_parent_keys(self):
1002
"""Return an iterable of keys of missing compression parents.
1004
Check this after calling insert_record_stream to find out if there are
1005
any missing compression parents. If there are, the records that
1006
depend on them are not able to be inserted safely. The precise
1007
behaviour depends on the concrete VersionedFiles class in use.
1009
Classes that do not support this will raise NotImplementedError.
1011
raise NotImplementedError(self.get_missing_compression_parent_keys)
1013
def insert_record_stream(self, stream):
1014
"""Insert a record stream into this container.
1016
:param stream: A stream of records to insert.
1018
:seealso VersionedFile.get_record_stream:
1020
raise NotImplementedError
1022
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1023
"""Iterate over the lines in the versioned files from keys.
1025
This may return lines from other keys. Each item the returned
1026
iterator yields is a tuple of a line and a text version that that line
1027
is present in (not introduced in).
1029
Ordering of results is in whatever order is most suitable for the
1030
underlying storage format.
1032
If a progress bar is supplied, it may be used to indicate progress.
1033
The caller is responsible for cleaning up progress bars (because this
1037
* Lines are normalised by the underlying store: they will all have \n
1039
* Lines are returned in arbitrary order.
1041
:return: An iterator over (line, key).
1043
raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
1046
"""Return a iterable of the keys for all the contained texts."""
1047
raise NotImplementedError(self.keys)
1049
def make_mpdiffs(self, keys):
1050
"""Create multiparent diffs for specified keys."""
1051
keys_order = tuple(keys)
1052
keys = frozenset(keys)
1053
knit_keys = set(keys)
1054
parent_map = self.get_parent_map(keys)
1055
for parent_keys in parent_map.itervalues():
1057
knit_keys.update(parent_keys)
1058
missing_keys = keys - set(parent_map)
1060
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
1061
# We need to filter out ghosts, because we can't diff against them.
1062
maybe_ghosts = knit_keys - keys
1063
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
1064
knit_keys.difference_update(ghosts)
1066
chunks_to_lines = osutils.chunks_to_lines
1067
for record in self.get_record_stream(knit_keys, 'topological', True):
1068
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
1069
# line_block_dict = {}
1070
# for parent, blocks in record.extract_line_blocks():
1071
# line_blocks[parent] = blocks
1072
# line_blocks[record.key] = line_block_dict
1074
for key in keys_order:
1076
parents = parent_map[key] or []
1077
# Note that filtering knit_keys can lead to a parent difference
1078
# between the creation and the application of the mpdiff.
1079
parent_lines = [lines[p] for p in parents if p in knit_keys]
1080
if len(parent_lines) > 0:
1081
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1084
left_parent_blocks = None
1085
diffs.append(multiparent.MultiParent.from_lines(target,
1086
parent_lines, left_parent_blocks))
1089
missing_keys = index._missing_keys_from_parent_map
1091
def _extract_blocks(self, version_id, source, target):
1095
class ThunkedVersionedFiles(VersionedFiles):
1096
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1098
This object allows a single keyspace for accessing the history graph and
1099
contents of named bytestrings.
1101
Currently no implementation allows the graph of different key prefixes to
1102
intersect, but the API does allow such implementations in the future.
1105
def __init__(self, transport, file_factory, mapper, is_locked):
1106
"""Create a ThunkedVersionedFiles."""
1107
self._transport = transport
1108
self._file_factory = file_factory
1109
self._mapper = mapper
1110
self._is_locked = is_locked
1112
def add_lines(self, key, parents, lines, parent_texts=None,
1113
left_matching_blocks=None, nostore_sha=None, random_id=False,
1114
check_content=True):
1115
"""See VersionedFiles.add_lines()."""
1116
path = self._mapper.map(key)
1117
version_id = key[-1]
1118
parents = [parent[-1] for parent in parents]
1119
vf = self._get_vf(path)
1122
return vf.add_lines_with_ghosts(version_id, parents, lines,
1123
parent_texts=parent_texts,
1124
left_matching_blocks=left_matching_blocks,
1125
nostore_sha=nostore_sha, random_id=random_id,
1126
check_content=check_content)
1127
except NotImplementedError:
1128
return vf.add_lines(version_id, parents, lines,
1129
parent_texts=parent_texts,
1130
left_matching_blocks=left_matching_blocks,
1131
nostore_sha=nostore_sha, random_id=random_id,
1132
check_content=check_content)
1133
except errors.NoSuchFile:
1134
# parent directory may be missing, try again.
1135
self._transport.mkdir(osutils.dirname(path))
1137
return vf.add_lines_with_ghosts(version_id, parents, lines,
1138
parent_texts=parent_texts,
1139
left_matching_blocks=left_matching_blocks,
1140
nostore_sha=nostore_sha, random_id=random_id,
1141
check_content=check_content)
1142
except NotImplementedError:
1143
return vf.add_lines(version_id, parents, lines,
1144
parent_texts=parent_texts,
1145
left_matching_blocks=left_matching_blocks,
1146
nostore_sha=nostore_sha, random_id=random_id,
1147
check_content=check_content)
1149
def annotate(self, key):
1150
"""Return a list of (version-key, line) tuples for the text of key.
1152
:raise RevisionNotPresent: If the key is not present.
1155
path = self._mapper.map(prefix)
1156
vf = self._get_vf(path)
1157
origins = vf.annotate(key[-1])
1159
for origin, line in origins:
1160
result.append((prefix + (origin,), line))
1163
def get_annotator(self):
1164
return annotate.Annotator(self)
1166
def check(self, progress_bar=None):
1167
"""See VersionedFiles.check()."""
1168
for prefix, vf in self._iter_all_components():
1171
def get_parent_map(self, keys):
1172
"""Get a map of the parents of keys.
1174
:param keys: The keys to look up parents for.
1175
:return: A mapping from keys to parents. Absent keys are absent from
1178
prefixes = self._partition_keys(keys)
1180
for prefix, suffixes in prefixes.items():
1181
path = self._mapper.map(prefix)
1182
vf = self._get_vf(path)
1183
parent_map = vf.get_parent_map(suffixes)
1184
for key, parents in parent_map.items():
1185
result[prefix + (key,)] = tuple(
1186
prefix + (parent,) for parent in parents)
1189
def _get_vf(self, path):
1190
if not self._is_locked():
1191
raise errors.ObjectNotLocked(self)
1192
return self._file_factory(path, self._transport, create=True,
1193
get_scope=lambda:None)
1195
def _partition_keys(self, keys):
1196
"""Turn keys into a dict of prefix:suffix_list."""
1199
prefix_keys = result.setdefault(key[:-1], [])
1200
prefix_keys.append(key[-1])
1203
def _get_all_prefixes(self):
1204
# Identify all key prefixes.
1205
# XXX: A bit hacky, needs polish.
1206
if type(self._mapper) == ConstantMapper:
1207
paths = [self._mapper.map(())]
1211
for quoted_relpath in self._transport.iter_files_recursive():
1212
path, ext = os.path.splitext(quoted_relpath)
1214
paths = list(relpaths)
1215
prefixes = [self._mapper.unmap(path) for path in paths]
1216
return zip(paths, prefixes)
1218
def get_record_stream(self, keys, ordering, include_delta_closure):
1219
"""See VersionedFiles.get_record_stream()."""
1220
# Ordering will be taken care of by each partitioned store; group keys
1223
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1224
suffixes = [(suffix,) for suffix in suffixes]
1225
for record in vf.get_record_stream(suffixes, ordering,
1226
include_delta_closure):
1227
if record.parents is not None:
1228
record.parents = tuple(
1229
prefix + parent for parent in record.parents)
1230
record.key = prefix + record.key
1233
def _iter_keys_vf(self, keys):
1234
prefixes = self._partition_keys(keys)
1236
for prefix, suffixes in prefixes.items():
1237
path = self._mapper.map(prefix)
1238
vf = self._get_vf(path)
1239
yield prefix, suffixes, vf
1241
def get_sha1s(self, keys):
1242
"""See VersionedFiles.get_sha1s()."""
1244
for prefix,suffixes, vf in self._iter_keys_vf(keys):
1245
vf_sha1s = vf.get_sha1s(suffixes)
1246
for suffix, sha1 in vf_sha1s.iteritems():
1247
sha1s[prefix + (suffix,)] = sha1
1250
def insert_record_stream(self, stream):
1251
"""Insert a record stream into this container.
1253
:param stream: A stream of records to insert.
1255
:seealso VersionedFile.get_record_stream:
1257
for record in stream:
1258
prefix = record.key[:-1]
1259
key = record.key[-1:]
1260
if record.parents is not None:
1261
parents = [parent[-1:] for parent in record.parents]
1264
thunk_record = AdapterFactory(key, parents, record)
1265
path = self._mapper.map(prefix)
1266
# Note that this parses the file many times; we can do better but
1267
# as this only impacts weaves in terms of performance, it is
1269
vf = self._get_vf(path)
1270
vf.insert_record_stream([thunk_record])
1272
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1273
"""Iterate over the lines in the versioned files from keys.
1275
This may return lines from other keys. Each item the returned
1276
iterator yields is a tuple of a line and a text version that that line
1277
is present in (not introduced in).
1279
Ordering of results is in whatever order is most suitable for the
1280
underlying storage format.
1282
If a progress bar is supplied, it may be used to indicate progress.
1283
The caller is responsible for cleaning up progress bars (because this
1287
* Lines are normalised by the underlying store: they will all have \n
1289
* Lines are returned in arbitrary order.
1291
:return: An iterator over (line, key).
1293
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1294
for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1295
yield line, prefix + (version,)
1297
def _iter_all_components(self):
1298
for path, prefix in self._get_all_prefixes():
1299
yield prefix, self._get_vf(path)
1302
"""See VersionedFiles.keys()."""
1304
for prefix, vf in self._iter_all_components():
1305
for suffix in vf.versions():
1306
result.add(prefix + (suffix,))
1310
class _PlanMergeVersionedFile(VersionedFiles):
1311
"""A VersionedFile for uncommitted and committed texts.
1313
It is intended to allow merges to be planned with working tree texts.
1314
It implements only the small part of the VersionedFiles interface used by
1315
PlanMerge. It falls back to multiple versionedfiles for data not stored in
1316
_PlanMergeVersionedFile itself.
1318
:ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1319
queried for missing texts.
1322
def __init__(self, file_id):
1323
"""Create a _PlanMergeVersionedFile.
1325
:param file_id: Used with _PlanMerge code which is not yet fully
1326
tuple-keyspace aware.
1328
self._file_id = file_id
1329
# fallback locations
1330
self.fallback_versionedfiles = []
1331
# Parents for locally held keys.
1333
# line data for locally held keys.
1335
# key lookup providers
1336
self._providers = [DictParentsProvider(self._parents)]
1338
def plan_merge(self, ver_a, ver_b, base=None):
1339
"""See VersionedFile.plan_merge"""
1340
from bzrlib.merge import _PlanMerge
1342
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1343
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1344
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1345
return _PlanMerge._subtract_plans(old_plan, new_plan)
1347
def plan_lca_merge(self, ver_a, ver_b, base=None):
1348
from bzrlib.merge import _PlanLCAMerge
1350
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1353
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1354
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1356
def add_lines(self, key, parents, lines):
1357
"""See VersionedFiles.add_lines
1359
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1360
are permitted. Only reserved ids are permitted.
1362
if type(key) is not tuple:
1363
raise TypeError(key)
1364
if not revision.is_reserved_id(key[-1]):
1365
raise ValueError('Only reserved ids may be used')
1367
raise ValueError('Parents may not be None')
1369
raise ValueError('Lines may not be None')
1370
self._parents[key] = tuple(parents)
1371
self._lines[key] = lines
1373
def get_record_stream(self, keys, ordering, include_delta_closure):
1376
if key in self._lines:
1377
lines = self._lines[key]
1378
parents = self._parents[key]
1380
yield ChunkedContentFactory(key, parents, None, lines)
1381
for versionedfile in self.fallback_versionedfiles:
1382
for record in versionedfile.get_record_stream(
1383
pending, 'unordered', True):
1384
if record.storage_kind == 'absent':
1387
pending.remove(record.key)
1391
# report absent entries
1393
yield AbsentContentFactory(key)
1395
def get_parent_map(self, keys):
1396
"""See VersionedFiles.get_parent_map"""
1397
# We create a new provider because a fallback may have been added.
1398
# If we make fallbacks private we can update a stack list and avoid
1399
# object creation thrashing.
1402
if revision.NULL_REVISION in keys:
1403
keys.remove(revision.NULL_REVISION)
1404
result[revision.NULL_REVISION] = ()
1405
self._providers = self._providers[:1] + self.fallback_versionedfiles
1407
StackedParentsProvider(self._providers).get_parent_map(keys))
1408
for key, parents in result.iteritems():
1410
result[key] = (revision.NULL_REVISION,)
1414
class PlanWeaveMerge(TextMerge):
1415
"""Weave merge that takes a plan as its input.
1417
This exists so that VersionedFile.plan_merge is implementable.
1418
Most callers will want to use WeaveMerge instead.
1421
def __init__(self, plan, a_marker=TextMerge.A_MARKER,
1422
b_marker=TextMerge.B_MARKER):
1423
TextMerge.__init__(self, a_marker, b_marker)
1426
def _merge_struct(self):
1431
def outstanding_struct():
1432
if not lines_a and not lines_b:
1434
elif ch_a and not ch_b:
1437
elif ch_b and not ch_a:
1439
elif lines_a == lines_b:
1442
yield (lines_a, lines_b)
1444
# We previously considered either 'unchanged' or 'killed-both' lines
1445
# to be possible places to resynchronize. However, assuming agreement
1446
# on killed-both lines may be too aggressive. -- mbp 20060324
1447
for state, line in self.plan:
1448
if state == 'unchanged':
1449
# resync and flush queued conflicts changes if any
1450
for struct in outstanding_struct():
1456
if state == 'unchanged':
1459
elif state == 'killed-a':
1461
lines_b.append(line)
1462
elif state == 'killed-b':
1464
lines_a.append(line)
1465
elif state == 'new-a':
1467
lines_a.append(line)
1468
elif state == 'new-b':
1470
lines_b.append(line)
1471
elif state == 'conflicted-a':
1473
lines_a.append(line)
1474
elif state == 'conflicted-b':
1476
lines_b.append(line)
1477
elif state == 'killed-both':
1478
# This counts as a change, even though there is no associated
1482
if state not in ('irrelevant', 'ghost-a', 'ghost-b',
1484
raise AssertionError(state)
1485
for struct in outstanding_struct():
1489
class WeaveMerge(PlanWeaveMerge):
1490
"""Weave merge that takes a VersionedFile and two versions as its input."""
1492
def __init__(self, versionedfile, ver_a, ver_b,
1493
a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1494
plan = versionedfile.plan_merge(ver_a, ver_b)
1495
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1498
class VirtualVersionedFiles(VersionedFiles):
1499
"""Dummy implementation for VersionedFiles that uses other functions for
1500
obtaining fulltexts and parent maps.
1502
This is always on the bottom of the stack and uses string keys
1503
(rather than tuples) internally.
1506
def __init__(self, get_parent_map, get_lines):
1507
"""Create a VirtualVersionedFiles.
1509
:param get_parent_map: Same signature as Repository.get_parent_map.
1510
:param get_lines: Should return lines for specified key or None if
1513
super(VirtualVersionedFiles, self).__init__()
1514
self._get_parent_map = get_parent_map
1515
self._get_lines = get_lines
1517
def check(self, progressbar=None):
1518
"""See VersionedFiles.check.
1520
:note: Always returns True for VirtualVersionedFiles.
1524
def add_mpdiffs(self, records):
1525
"""See VersionedFiles.mpdiffs.
1527
:note: Not implemented for VirtualVersionedFiles.
1529
raise NotImplementedError(self.add_mpdiffs)
1531
def get_parent_map(self, keys):
1532
"""See VersionedFiles.get_parent_map."""
1533
return dict([((k,), tuple([(p,) for p in v]))
1534
for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
1536
def get_sha1s(self, keys):
1537
"""See VersionedFiles.get_sha1s."""
1540
lines = self._get_lines(k)
1541
if lines is not None:
1542
if not isinstance(lines, list):
1543
raise AssertionError
1544
ret[(k,)] = osutils.sha_strings(lines)
1547
def get_record_stream(self, keys, ordering, include_delta_closure):
1548
"""See VersionedFiles.get_record_stream."""
1549
for (k,) in list(keys):
1550
lines = self._get_lines(k)
1551
if lines is not None:
1552
if not isinstance(lines, list):
1553
raise AssertionError
1554
yield ChunkedContentFactory((k,), None,
1555
sha1=osutils.sha_strings(lines),
1558
yield AbsentContentFactory((k,))
1560
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1561
"""See VersionedFile.iter_lines_added_or_present_in_versions()."""
1562
for i, (key,) in enumerate(keys):
1564
pb.update("Finding changed lines", i, len(keys))
1565
for l in self._get_lines(key):
1569
def network_bytes_to_kind_and_offset(network_bytes):
1570
"""Strip of a record kind from the front of network_bytes.
1572
:param network_bytes: The bytes of a record.
1573
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1575
line_end = network_bytes.find('\n')
1576
storage_kind = network_bytes[:line_end]
1577
return storage_kind, line_end + 1
1580
class NetworkRecordStream(object):
1581
"""A record_stream which reconstitures a serialised stream."""
1583
def __init__(self, bytes_iterator):
1584
"""Create a NetworkRecordStream.
1586
:param bytes_iterator: An iterator of bytes. Each item in this
1587
iterator should have been obtained from a record_streams'
1588
record.get_bytes_as(record.storage_kind) call.
1590
self._bytes_iterator = bytes_iterator
1591
self._kind_factory = {
1592
'fulltext': fulltext_network_to_record,
1593
'groupcompress-block': groupcompress.network_block_to_records,
1594
'inventory-delta': inventory_delta_network_to_record,
1595
'knit-ft-gz': knit.knit_network_to_record,
1596
'knit-delta-gz': knit.knit_network_to_record,
1597
'knit-annotated-ft-gz': knit.knit_network_to_record,
1598
'knit-annotated-delta-gz': knit.knit_network_to_record,
1599
'knit-delta-closure': knit.knit_delta_closure_to_records,
1605
:return: An iterator as per VersionedFiles.get_record_stream().
1607
for bytes in self._bytes_iterator:
1608
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1609
for record in self._kind_factory[storage_kind](
1610
storage_kind, bytes, line_end):
1614
def fulltext_network_to_record(kind, bytes, line_end):
1615
"""Convert a network fulltext record to record."""
1616
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1617
record_meta = bytes[line_end+4:line_end+4+meta_len]
1618
key, parents = bencode.bdecode_as_tuple(record_meta)
1619
if parents == 'nil':
1621
fulltext = bytes[line_end+4+meta_len:]
1622
return [FulltextContentFactory(key, parents, None, fulltext)]
1625
def inventory_delta_network_to_record(kind, bytes, line_end):
1626
"""Convert a network inventory-delta record to record."""
1627
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1628
record_meta = bytes[line_end+4:line_end+4+meta_len]
1629
key, parents = bencode.bdecode_as_tuple(record_meta)
1630
if parents == 'nil':
1632
inventory_delta_bytes = bytes[line_end+4+meta_len:]
1633
deserialiser = inventory_delta.InventoryDeltaSerializer()
1634
parse_result = deserialiser.parse_text_bytes(inventory_delta_bytes)
1635
basis_id, new_id, rich_root, tree_refs, delta = parse_result
1636
return [InventoryDeltaContentFactory(
1637
key, parents, None, delta, basis_id, (rich_root, tree_refs))]
1640
def _length_prefix(bytes):
1641
return struct.pack('!L', len(bytes))
1644
def record_to_fulltext_bytes(record):
1645
if record.parents is None:
1648
parents = record.parents
1649
record_meta = bencode.bencode((record.key, parents))
1650
record_content = record.get_bytes_as('fulltext')
1651
return "fulltext\n%s%s%s" % (
1652
_length_prefix(record_meta), record_meta, record_content)
1655
def record_to_inventory_delta_bytes(record):
1656
record_content = record.get_bytes_as('inventory-delta-bytes')
1657
if record.parents is None:
1660
parents = record.parents
1661
record_meta = bencode.bencode((record.key, parents))
1662
return "inventory-delta\n%s%s%s" % (
1663
_length_prefix(record_meta), record_meta, record_content)
1666
def sort_groupcompress(parent_map):
1667
"""Sort and group the keys in parent_map into groupcompress order.
1669
groupcompress is defined (currently) as reverse-topological order, grouped
1672
:return: A sorted-list of keys
1674
# gc-optimal ordering is approximately reverse topological,
1675
# properly grouped by file-id.
1677
for item in parent_map.iteritems():
1679
if isinstance(key, str) or len(key) == 1:
1684
per_prefix_map[prefix].append(item)
1686
per_prefix_map[prefix] = [item]
1689
for prefix in sorted(per_prefix_map):
1690
present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
added
removed
bzrlib/versionedfile.py
