1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20
"""Versioned text file storage api."""
23
from cStringIO import StringIO
26
from zlib import adler32
28
from bzrlib.lazy_import import lazy_import
29
lazy_import(globals(), """
42
from bzrlib.graph import DictParentsProvider, Graph, _StackedParentsProvider
43
from bzrlib.transport.memory import MemoryTransport
45
from bzrlib.inter import InterObject
46
from bzrlib.registry import Registry
47
from bzrlib.symbol_versioning import *
48
from bzrlib.textmerge import TextMerge
49
from bzrlib.util import bencode
52
adapter_registry = Registry()
53
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
54
'DeltaPlainToFullText')
55
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
57
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
58
'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
59
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
60
'bzrlib.knit', 'DeltaAnnotatedToFullText')
61
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
62
'bzrlib.knit', 'FTAnnotatedToUnannotated')
63
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
64
'bzrlib.knit', 'FTAnnotatedToFullText')
65
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
66
# 'bzrlib.knit', 'FTAnnotatedToChunked')
69
class ContentFactory(object):
70
"""Abstract interface for insertion and retrieval from a VersionedFile.
72
:ivar sha1: None, or the sha1 of the content fulltext.
73
:ivar storage_kind: The native storage kind of this factory. One of
74
'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
75
'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
76
'knit-annotated-delta-gz', 'knit-ft-gz', 'knit-delta-gz'.
77
:ivar key: The key of this content. Each key is a tuple with a single
79
:ivar parents: A tuple of parent keys for self.key. If the object has
80
no parent information, None (as opposed to () for an empty list of
85
"""Create a ContentFactory."""
87
self.storage_kind = None
92
class ChunkedContentFactory(ContentFactory):
93
"""Static data content factory.
95
This takes a 'chunked' list of strings. The only requirement on 'chunked' is
96
that ''.join(lines) becomes a valid fulltext. A tuple of a single string
97
satisfies this, as does a list of lines.
99
:ivar sha1: None, or the sha1 of the content fulltext.
100
:ivar storage_kind: The native storage kind of this factory. Always
102
:ivar key: The key of this content. Each key is a tuple with a single
104
:ivar parents: A tuple of parent keys for self.key. If the object has
105
no parent information, None (as opposed to () for an empty list of
109
def __init__(self, key, parents, sha1, chunks):
110
"""Create a ContentFactory."""
112
self.storage_kind = 'chunked'
114
self.parents = parents
115
self._chunks = chunks
117
def get_bytes_as(self, storage_kind):
118
if storage_kind == 'chunked':
120
elif storage_kind == 'fulltext':
121
return ''.join(self._chunks)
122
raise errors.UnavailableRepresentation(self.key, storage_kind,
126
class FulltextContentFactory(ContentFactory):
127
"""Static data content factory.
129
This takes a fulltext when created and just returns that during
130
get_bytes_as('fulltext').
132
:ivar sha1: None, or the sha1 of the content fulltext.
133
:ivar storage_kind: The native storage kind of this factory. Always
135
:ivar key: The key of this content. Each key is a tuple with a single
137
:ivar parents: A tuple of parent keys for self.key. If the object has
138
no parent information, None (as opposed to () for an empty list of
142
def __init__(self, key, parents, sha1, text):
143
"""Create a ContentFactory."""
145
self.storage_kind = 'fulltext'
147
self.parents = parents
150
def get_bytes_as(self, storage_kind):
151
if storage_kind == self.storage_kind:
153
elif storage_kind == 'chunked':
155
raise errors.UnavailableRepresentation(self.key, storage_kind,
159
class AbsentContentFactory(ContentFactory):
160
"""A placeholder content factory for unavailable texts.
163
:ivar storage_kind: 'absent'.
164
:ivar key: The key of this content. Each key is a tuple with a single
169
def __init__(self, key):
170
"""Create a ContentFactory."""
172
self.storage_kind = 'absent'
177
class AdapterFactory(ContentFactory):
178
"""A content factory to adapt between key prefix's."""
180
def __init__(self, key, parents, adapted):
181
"""Create an adapter factory instance."""
183
self.parents = parents
184
self._adapted = adapted
186
def __getattr__(self, attr):
187
"""Return a member from the adapted object."""
188
if attr in ('key', 'parents'):
189
return self.__dict__[attr]
191
return getattr(self._adapted, attr)
194
def filter_absent(record_stream):
195
"""Adapt a record stream to remove absent records."""
196
for record in record_stream:
197
if record.storage_kind != 'absent':
201
class VersionedFile(object):
202
"""Versioned text file storage.
204
A versioned file manages versions of line-based text files,
205
keeping track of the originating version for each line.
207
To clients the "lines" of the file are represented as a list of
208
strings. These strings will typically have terminal newline
209
characters, but this is not required. In particular files commonly
210
do not have a newline at the end of the file.
212
Texts are identified by a version-id string.
216
def check_not_reserved_id(version_id):
217
revision.check_not_reserved_id(version_id)
219
def copy_to(self, name, transport):
220
"""Copy this versioned file to name on transport."""
221
raise NotImplementedError(self.copy_to)
223
def get_record_stream(self, versions, ordering, include_delta_closure):
224
"""Get a stream of records for versions.
226
:param versions: The versions to include. Each version is a tuple
228
:param ordering: Either 'unordered' or 'topological'. A topologically
229
sorted stream has compression parents strictly before their
231
:param include_delta_closure: If True then the closure across any
232
compression parents will be included (in the data content of the
233
stream, not in the emitted records). This guarantees that
234
'fulltext' can be used successfully on every record.
235
:return: An iterator of ContentFactory objects, each of which is only
236
valid until the iterator is advanced.
238
raise NotImplementedError(self.get_record_stream)
240
def has_version(self, version_id):
241
"""Returns whether version is present."""
242
raise NotImplementedError(self.has_version)
244
def insert_record_stream(self, stream):
245
"""Insert a record stream into this versioned file.
247
:param stream: A stream of records to insert.
249
:seealso VersionedFile.get_record_stream:
251
raise NotImplementedError
253
def add_lines(self, version_id, parents, lines, parent_texts=None,
254
left_matching_blocks=None, nostore_sha=None, random_id=False,
256
"""Add a single text on top of the versioned file.
258
Must raise RevisionAlreadyPresent if the new version is
259
already present in file history.
261
Must raise RevisionNotPresent if any of the given parents are
262
not present in file history.
264
:param lines: A list of lines. Each line must be a bytestring. And all
265
of them except the last must be terminated with \n and contain no
266
other \n's. The last line may either contain no \n's or a single
267
terminated \n. If the lines list does meet this constraint the add
268
routine may error or may succeed - but you will be unable to read
269
the data back accurately. (Checking the lines have been split
270
correctly is expensive and extremely unlikely to catch bugs so it
271
is not done at runtime unless check_content is True.)
272
:param parent_texts: An optional dictionary containing the opaque
273
representations of some or all of the parents of version_id to
274
allow delta optimisations. VERY IMPORTANT: the texts must be those
275
returned by add_lines or data corruption can be caused.
276
:param left_matching_blocks: a hint about which areas are common
277
between the text and its left-hand-parent. The format is
278
the SequenceMatcher.get_matching_blocks format.
279
:param nostore_sha: Raise ExistingContent and do not add the lines to
280
the versioned file if the digest of the lines matches this.
281
:param random_id: If True a random id has been selected rather than
282
an id determined by some deterministic process such as a converter
283
from a foreign VCS. When True the backend may choose not to check
284
for uniqueness of the resulting key within the versioned file, so
285
this should only be done when the result is expected to be unique
287
:param check_content: If True, the lines supplied are verified to be
288
bytestrings that are correctly formed lines.
289
:return: The text sha1, the number of bytes in the text, and an opaque
290
representation of the inserted version which can be provided
291
back to future add_lines calls in the parent_texts dictionary.
293
self._check_write_ok()
294
return self._add_lines(version_id, parents, lines, parent_texts,
295
left_matching_blocks, nostore_sha, random_id, check_content)
297
def _add_lines(self, version_id, parents, lines, parent_texts,
298
left_matching_blocks, nostore_sha, random_id, check_content):
299
"""Helper to do the class specific add_lines."""
300
raise NotImplementedError(self.add_lines)
302
def add_lines_with_ghosts(self, version_id, parents, lines,
303
parent_texts=None, nostore_sha=None, random_id=False,
304
check_content=True, left_matching_blocks=None):
305
"""Add lines to the versioned file, allowing ghosts to be present.
307
This takes the same parameters as add_lines and returns the same.
309
self._check_write_ok()
310
return self._add_lines_with_ghosts(version_id, parents, lines,
311
parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
313
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
314
nostore_sha, random_id, check_content, left_matching_blocks):
315
"""Helper to do class specific add_lines_with_ghosts."""
316
raise NotImplementedError(self.add_lines_with_ghosts)
318
def check(self, progress_bar=None):
319
"""Check the versioned file for integrity."""
320
raise NotImplementedError(self.check)
322
def _check_lines_not_unicode(self, lines):
323
"""Check that lines being added to a versioned file are not unicode."""
325
if line.__class__ is not str:
326
raise errors.BzrBadParameterUnicode("lines")
328
def _check_lines_are_lines(self, lines):
329
"""Check that the lines really are full lines without inline EOL."""
331
if '\n' in line[:-1]:
332
raise errors.BzrBadParameterContainsNewline("lines")
334
def get_format_signature(self):
335
"""Get a text description of the data encoding in this file.
339
raise NotImplementedError(self.get_format_signature)
341
def make_mpdiffs(self, version_ids):
342
"""Create multiparent diffs for specified versions."""
343
knit_versions = set()
344
knit_versions.update(version_ids)
345
parent_map = self.get_parent_map(version_ids)
346
for version_id in version_ids:
348
knit_versions.update(parent_map[version_id])
350
raise errors.RevisionNotPresent(version_id, self)
351
# We need to filter out ghosts, because we can't diff against them.
352
knit_versions = set(self.get_parent_map(knit_versions).keys())
353
lines = dict(zip(knit_versions,
354
self._get_lf_split_line_list(knit_versions)))
356
for version_id in version_ids:
357
target = lines[version_id]
359
parents = [lines[p] for p in parent_map[version_id] if p in
362
# I don't know how this could ever trigger.
363
# parent_map[version_id] was already triggered in the previous
364
# for loop, and lines[p] has the 'if p in knit_versions' check,
365
# so we again won't have a KeyError.
366
raise errors.RevisionNotPresent(version_id, self)
368
left_parent_blocks = self._extract_blocks(version_id,
371
left_parent_blocks = None
372
diffs.append(multiparent.MultiParent.from_lines(target, parents,
376
def _extract_blocks(self, version_id, source, target):
379
def add_mpdiffs(self, records):
380
"""Add mpdiffs to this VersionedFile.
382
Records should be iterables of version, parents, expected_sha1,
383
mpdiff. mpdiff should be a MultiParent instance.
385
# Does this need to call self._check_write_ok()? (IanC 20070919)
387
mpvf = multiparent.MultiMemoryVersionedFile()
389
for version, parent_ids, expected_sha1, mpdiff in records:
390
versions.append(version)
391
mpvf.add_diff(mpdiff, version, parent_ids)
392
needed_parents = set()
393
for version, parent_ids, expected_sha1, mpdiff in records:
394
needed_parents.update(p for p in parent_ids
395
if not mpvf.has_version(p))
396
present_parents = set(self.get_parent_map(needed_parents).keys())
397
for parent_id, lines in zip(present_parents,
398
self._get_lf_split_line_list(present_parents)):
399
mpvf.add_version(lines, parent_id, [])
400
for (version, parent_ids, expected_sha1, mpdiff), lines in\
401
zip(records, mpvf.get_line_list(versions)):
402
if len(parent_ids) == 1:
403
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
404
mpvf.get_diff(parent_ids[0]).num_lines()))
406
left_matching_blocks = None
408
_, _, version_text = self.add_lines_with_ghosts(version,
409
parent_ids, lines, vf_parents,
410
left_matching_blocks=left_matching_blocks)
411
except NotImplementedError:
412
# The vf can't handle ghosts, so add lines normally, which will
413
# (reasonably) fail if there are ghosts in the data.
414
_, _, version_text = self.add_lines(version,
415
parent_ids, lines, vf_parents,
416
left_matching_blocks=left_matching_blocks)
417
vf_parents[version] = version_text
418
sha1s = self.get_sha1s(versions)
419
for version, parent_ids, expected_sha1, mpdiff in records:
420
if expected_sha1 != sha1s[version]:
421
raise errors.VersionedFileInvalidChecksum(version)
423
def get_text(self, version_id):
424
"""Return version contents as a text string.
426
Raises RevisionNotPresent if version is not present in
429
return ''.join(self.get_lines(version_id))
430
get_string = get_text
432
def get_texts(self, version_ids):
433
"""Return the texts of listed versions as a list of strings.
435
Raises RevisionNotPresent if version is not present in
438
return [''.join(self.get_lines(v)) for v in version_ids]
440
def get_lines(self, version_id):
441
"""Return version contents as a sequence of lines.
443
Raises RevisionNotPresent if version is not present in
446
raise NotImplementedError(self.get_lines)
448
def _get_lf_split_line_list(self, version_ids):
449
return [StringIO(t).readlines() for t in self.get_texts(version_ids)]
451
def get_ancestry(self, version_ids, topo_sorted=True):
452
"""Return a list of all ancestors of given version(s). This
453
will not include the null revision.
455
This list will not be topologically sorted if topo_sorted=False is
458
Must raise RevisionNotPresent if any of the given versions are
459
not present in file history."""
460
if isinstance(version_ids, basestring):
461
version_ids = [version_ids]
462
raise NotImplementedError(self.get_ancestry)
464
def get_ancestry_with_ghosts(self, version_ids):
465
"""Return a list of all ancestors of given version(s). This
466
will not include the null revision.
468
Must raise RevisionNotPresent if any of the given versions are
469
not present in file history.
471
Ghosts that are known about will be included in ancestry list,
472
but are not explicitly marked.
474
raise NotImplementedError(self.get_ancestry_with_ghosts)
476
def get_parent_map(self, version_ids):
477
"""Get a map of the parents of version_ids.
479
:param version_ids: The version ids to look up parents for.
480
:return: A mapping from version id to parents.
482
raise NotImplementedError(self.get_parent_map)
484
def get_parents_with_ghosts(self, version_id):
485
"""Return version names for parents of version_id.
487
Will raise RevisionNotPresent if version_id is not present
490
Ghosts that are known about will be included in the parent list,
491
but are not explicitly marked.
494
return list(self.get_parent_map([version_id])[version_id])
496
raise errors.RevisionNotPresent(version_id, self)
498
def annotate(self, version_id):
499
"""Return a list of (version-id, line) tuples for version_id.
501
:raise RevisionNotPresent: If the given version is
502
not present in file history.
504
raise NotImplementedError(self.annotate)
506
def iter_lines_added_or_present_in_versions(self, version_ids=None,
508
"""Iterate over the lines in the versioned file from version_ids.
510
This may return lines from other versions. Each item the returned
511
iterator yields is a tuple of a line and a text version that that line
512
is present in (not introduced in).
514
Ordering of results is in whatever order is most suitable for the
515
underlying storage format.
517
If a progress bar is supplied, it may be used to indicate progress.
518
The caller is responsible for cleaning up progress bars (because this
521
NOTES: Lines are normalised: they will all have \n terminators.
522
Lines are returned in arbitrary order.
524
:return: An iterator over (line, version_id).
526
raise NotImplementedError(self.iter_lines_added_or_present_in_versions)
528
def plan_merge(self, ver_a, ver_b):
529
"""Return pseudo-annotation indicating how the two versions merge.
531
This is computed between versions a and b and their common
534
Weave lines present in none of them are skipped entirely.
537
killed-base Dead in base revision
538
killed-both Killed in each revision
541
unchanged Alive in both a and b (possibly created in both)
544
ghost-a Killed in a, unborn in b
545
ghost-b Killed in b, unborn in a
546
irrelevant Not in either revision
548
raise NotImplementedError(VersionedFile.plan_merge)
550
def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
551
b_marker=TextMerge.B_MARKER):
552
return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
555
class RecordingVersionedFilesDecorator(object):
556
"""A minimal versioned files that records calls made on it.
558
Only enough methods have been added to support tests using it to date.
560
:ivar calls: A list of the calls made; can be reset at any time by
564
def __init__(self, backing_vf):
565
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
567
:param backing_vf: The versioned file to answer all methods.
569
self._backing_vf = backing_vf
572
def add_lines(self, key, parents, lines, parent_texts=None,
573
left_matching_blocks=None, nostore_sha=None, random_id=False,
575
self.calls.append(("add_lines", key, parents, lines, parent_texts,
576
left_matching_blocks, nostore_sha, random_id, check_content))
577
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
578
left_matching_blocks, nostore_sha, random_id, check_content)
581
self._backing_vf.check()
583
def get_parent_map(self, keys):
584
self.calls.append(("get_parent_map", copy(keys)))
585
return self._backing_vf.get_parent_map(keys)
587
def get_record_stream(self, keys, sort_order, include_delta_closure):
588
self.calls.append(("get_record_stream", list(keys), sort_order,
589
include_delta_closure))
590
return self._backing_vf.get_record_stream(keys, sort_order,
591
include_delta_closure)
593
def get_sha1s(self, keys):
594
self.calls.append(("get_sha1s", copy(keys)))
595
return self._backing_vf.get_sha1s(keys)
597
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
598
self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
599
return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
602
self.calls.append(("keys",))
603
return self._backing_vf.keys()
606
class OrderingVersionedFilesDecorator(RecordingVersionedFilesDecorator):
607
"""A VF that records calls, and returns keys in specific order.
609
:ivar calls: A list of the calls made; can be reset at any time by
613
def __init__(self, backing_vf, key_priority):
614
"""Create a RecordingVersionedFilesDecorator decorating backing_vf.
616
:param backing_vf: The versioned file to answer all methods.
617
:param key_priority: A dictionary defining what order keys should be
618
returned from an 'unordered' get_record_stream request.
619
Keys with lower priority are returned first, keys not present in
620
the map get an implicit priority of 0, and are returned in
621
lexicographical order.
623
RecordingVersionedFilesDecorator.__init__(self, backing_vf)
624
self._key_priority = key_priority
626
def get_record_stream(self, keys, sort_order, include_delta_closure):
627
self.calls.append(("get_record_stream", list(keys), sort_order,
628
include_delta_closure))
629
if sort_order == 'unordered':
631
return (self._key_priority.get(key, 0), key)
632
# Use a defined order by asking for the keys one-by-one from the
634
for key in sorted(keys, key=sort_key):
635
for record in self._backing_vf.get_record_stream([key],
636
'unordered', include_delta_closure):
639
for record in self._backing_vf.get_record_stream(keys, sort_order,
640
include_delta_closure):
644
class KeyMapper(object):
645
"""KeyMappers map between keys and underlying partitioned storage."""
648
"""Map key to an underlying storage identifier.
650
:param key: A key tuple e.g. ('file-id', 'revision-id').
651
:return: An underlying storage identifier, specific to the partitioning
654
raise NotImplementedError(self.map)
656
def unmap(self, partition_id):
657
"""Map a partitioned storage id back to a key prefix.
659
:param partition_id: The underlying partition id.
660
:return: As much of a key (or prefix) as is derivable from the partition
663
raise NotImplementedError(self.unmap)
666
class ConstantMapper(KeyMapper):
667
"""A key mapper that maps to a constant result."""
669
def __init__(self, result):
670
"""Create a ConstantMapper which will return result for all maps."""
671
self._result = result
674
"""See KeyMapper.map()."""
678
class URLEscapeMapper(KeyMapper):
679
"""Base class for use with transport backed storage.
681
This provides a map and unmap wrapper that respectively url escape and
682
unescape their outputs and inputs.
686
"""See KeyMapper.map()."""
687
return urllib.quote(self._map(key))
689
def unmap(self, partition_id):
690
"""See KeyMapper.unmap()."""
691
return self._unmap(urllib.unquote(partition_id))
694
class PrefixMapper(URLEscapeMapper):
695
"""A key mapper that extracts the first component of a key.
697
This mapper is for use with a transport based backend.
701
"""See KeyMapper.map()."""
704
def _unmap(self, partition_id):
705
"""See KeyMapper.unmap()."""
706
return (partition_id,)
709
class HashPrefixMapper(URLEscapeMapper):
710
"""A key mapper that combines the first component of a key with a hash.
712
This mapper is for use with a transport based backend.
716
"""See KeyMapper.map()."""
717
prefix = self._escape(key[0])
718
return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
720
def _escape(self, prefix):
721
"""No escaping needed here."""
724
def _unmap(self, partition_id):
725
"""See KeyMapper.unmap()."""
726
return (self._unescape(osutils.basename(partition_id)),)
728
def _unescape(self, basename):
729
"""No unescaping needed for HashPrefixMapper."""
733
class HashEscapedPrefixMapper(HashPrefixMapper):
734
"""Combines the escaped first component of a key with a hash.
736
This mapper is for use with a transport based backend.
739
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
741
def _escape(self, prefix):
742
"""Turn a key element into a filesystem safe string.
744
This is similar to a plain urllib.quote, except
745
it uses specific safe characters, so that it doesn't
746
have to translate a lot of valid file ids.
748
# @ does not get escaped. This is because it is a valid
749
# filesystem character we use all the time, and it looks
750
# a lot better than seeing %40 all the time.
751
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
755
def _unescape(self, basename):
756
"""Escaped names are easily unescaped by urlutils."""
757
return urllib.unquote(basename)
760
def make_versioned_files_factory(versioned_file_factory, mapper):
761
"""Create a ThunkedVersionedFiles factory.
763
This will create a callable which when called creates a
764
ThunkedVersionedFiles on a transport, using mapper to access individual
765
versioned files, and versioned_file_factory to create each individual file.
767
def factory(transport):
768
return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
773
class VersionedFiles(object):
774
"""Storage for many versioned files.
776
This object allows a single keyspace for accessing the history graph and
777
contents of named bytestrings.
779
Currently no implementation allows the graph of different key prefixes to
780
intersect, but the API does allow such implementations in the future.
782
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
783
may have a different length key-size, but that size will be constant for
784
all texts added to or retrieved from it. For instance, bzrlib uses
785
instances with a key-size of 2 for storing user files in a repository, with
786
the first element the fileid, and the second the version of that file.
788
The use of tuples allows a single code base to support several different
789
uses with only the mapping logic changing from instance to instance.
792
def add_lines(self, key, parents, lines, parent_texts=None,
793
left_matching_blocks=None, nostore_sha=None, random_id=False,
795
"""Add a text to the store.
797
:param key: The key tuple of the text to add.
798
:param parents: The parents key tuples of the text to add.
799
:param lines: A list of lines. Each line must be a bytestring. And all
800
of them except the last must be terminated with \n and contain no
801
other \n's. The last line may either contain no \n's or a single
802
terminating \n. If the lines list does meet this constraint the add
803
routine may error or may succeed - but you will be unable to read
804
the data back accurately. (Checking the lines have been split
805
correctly is expensive and extremely unlikely to catch bugs so it
806
is not done at runtime unless check_content is True.)
807
:param parent_texts: An optional dictionary containing the opaque
808
representations of some or all of the parents of version_id to
809
allow delta optimisations. VERY IMPORTANT: the texts must be those
810
returned by add_lines or data corruption can be caused.
811
:param left_matching_blocks: a hint about which areas are common
812
between the text and its left-hand-parent. The format is
813
the SequenceMatcher.get_matching_blocks format.
814
:param nostore_sha: Raise ExistingContent and do not add the lines to
815
the versioned file if the digest of the lines matches this.
816
:param random_id: If True a random id has been selected rather than
817
an id determined by some deterministic process such as a converter
818
from a foreign VCS. When True the backend may choose not to check
819
for uniqueness of the resulting key within the versioned file, so
820
this should only be done when the result is expected to be unique
822
:param check_content: If True, the lines supplied are verified to be
823
bytestrings that are correctly formed lines.
824
:return: The text sha1, the number of bytes in the text, and an opaque
825
representation of the inserted version which can be provided
826
back to future add_lines calls in the parent_texts dictionary.
828
raise NotImplementedError(self.add_lines)
830
def add_mpdiffs(self, records):
831
"""Add mpdiffs to this VersionedFile.
833
Records should be iterables of version, parents, expected_sha1,
834
mpdiff. mpdiff should be a MultiParent instance.
837
mpvf = multiparent.MultiMemoryVersionedFile()
839
for version, parent_ids, expected_sha1, mpdiff in records:
840
versions.append(version)
841
mpvf.add_diff(mpdiff, version, parent_ids)
842
needed_parents = set()
843
for version, parent_ids, expected_sha1, mpdiff in records:
844
needed_parents.update(p for p in parent_ids
845
if not mpvf.has_version(p))
846
# It seems likely that adding all the present parents as fulltexts can
847
# easily exhaust memory.
848
chunks_to_lines = osutils.chunks_to_lines
849
for record in self.get_record_stream(needed_parents, 'unordered',
851
if record.storage_kind == 'absent':
853
mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
855
for (key, parent_keys, expected_sha1, mpdiff), lines in\
856
zip(records, mpvf.get_line_list(versions)):
857
if len(parent_keys) == 1:
858
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
859
mpvf.get_diff(parent_keys[0]).num_lines()))
861
left_matching_blocks = None
862
version_sha1, _, version_text = self.add_lines(key,
863
parent_keys, lines, vf_parents,
864
left_matching_blocks=left_matching_blocks)
865
if version_sha1 != expected_sha1:
866
raise errors.VersionedFileInvalidChecksum(version)
867
vf_parents[key] = version_text
869
def annotate(self, key):
870
"""Return a list of (version-key, line) tuples for the text of key.
872
:raise RevisionNotPresent: If the key is not present.
874
raise NotImplementedError(self.annotate)
876
def check(self, progress_bar=None):
877
"""Check this object for integrity."""
878
raise NotImplementedError(self.check)
881
def check_not_reserved_id(version_id):
882
revision.check_not_reserved_id(version_id)
884
def _check_lines_not_unicode(self, lines):
885
"""Check that lines being added to a versioned file are not unicode."""
887
if line.__class__ is not str:
888
raise errors.BzrBadParameterUnicode("lines")
890
def _check_lines_are_lines(self, lines):
891
"""Check that the lines really are full lines without inline EOL."""
893
if '\n' in line[:-1]:
894
raise errors.BzrBadParameterContainsNewline("lines")
896
def get_parent_map(self, keys):
897
"""Get a map of the parents of keys.
899
:param keys: The keys to look up parents for.
900
:return: A mapping from keys to parents. Absent keys are absent from
903
raise NotImplementedError(self.get_parent_map)
905
def get_record_stream(self, keys, ordering, include_delta_closure):
906
"""Get a stream of records for keys.
908
:param keys: The keys to include.
909
:param ordering: Either 'unordered' or 'topological'. A topologically
910
sorted stream has compression parents strictly before their
912
:param include_delta_closure: If True then the closure across any
913
compression parents will be included (in the opaque data).
914
:return: An iterator of ContentFactory objects, each of which is only
915
valid until the iterator is advanced.
917
raise NotImplementedError(self.get_record_stream)
919
def get_sha1s(self, keys):
920
"""Get the sha1's of the texts for the given keys.
922
:param keys: The names of the keys to lookup
923
:return: a dict from key to sha1 digest. Keys of texts which are not
924
present in the store are not present in the returned
927
raise NotImplementedError(self.get_sha1s)
929
has_key = index._has_key_from_parent_map
931
def insert_record_stream(self, stream):
932
"""Insert a record stream into this container.
934
:param stream: A stream of records to insert.
936
:seealso VersionedFile.get_record_stream:
938
raise NotImplementedError
940
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
941
"""Iterate over the lines in the versioned files from keys.
943
This may return lines from other keys. Each item the returned
944
iterator yields is a tuple of a line and a text version that that line
945
is present in (not introduced in).
947
Ordering of results is in whatever order is most suitable for the
948
underlying storage format.
950
If a progress bar is supplied, it may be used to indicate progress.
951
The caller is responsible for cleaning up progress bars (because this
955
* Lines are normalised by the underlying store: they will all have \n
957
* Lines are returned in arbitrary order.
959
:return: An iterator over (line, key).
961
raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
964
"""Return a iterable of the keys for all the contained texts."""
965
raise NotImplementedError(self.keys)
967
def make_mpdiffs(self, keys):
968
"""Create multiparent diffs for specified keys."""
969
keys_order = tuple(keys)
970
keys = frozenset(keys)
971
knit_keys = set(keys)
972
parent_map = self.get_parent_map(keys)
973
for parent_keys in parent_map.itervalues():
975
knit_keys.update(parent_keys)
976
missing_keys = keys - set(parent_map)
978
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
979
# We need to filter out ghosts, because we can't diff against them.
980
maybe_ghosts = knit_keys - keys
981
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
982
knit_keys.difference_update(ghosts)
984
chunks_to_lines = osutils.chunks_to_lines
985
for record in self.get_record_stream(knit_keys, 'topological', True):
986
lines[record.key] = chunks_to_lines(record.get_bytes_as('chunked'))
987
# line_block_dict = {}
988
# for parent, blocks in record.extract_line_blocks():
989
# line_blocks[parent] = blocks
990
# line_blocks[record.key] = line_block_dict
992
for key in keys_order:
994
parents = parent_map[key] or []
995
# Note that filtering knit_keys can lead to a parent difference
996
# between the creation and the application of the mpdiff.
997
parent_lines = [lines[p] for p in parents if p in knit_keys]
998
if len(parent_lines) > 0:
999
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
1002
left_parent_blocks = None
1003
diffs.append(multiparent.MultiParent.from_lines(target,
1004
parent_lines, left_parent_blocks))
1007
missing_keys = index._missing_keys_from_parent_map
1009
def _extract_blocks(self, version_id, source, target):
1013
class ThunkedVersionedFiles(VersionedFiles):
1014
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
1016
This object allows a single keyspace for accessing the history graph and
1017
contents of named bytestrings.
1019
Currently no implementation allows the graph of different key prefixes to
1020
intersect, but the API does allow such implementations in the future.
1023
def __init__(self, transport, file_factory, mapper, is_locked):
1024
"""Create a ThunkedVersionedFiles."""
1025
self._transport = transport
1026
self._file_factory = file_factory
1027
self._mapper = mapper
1028
self._is_locked = is_locked
1030
def add_lines(self, key, parents, lines, parent_texts=None,
1031
left_matching_blocks=None, nostore_sha=None, random_id=False,
1032
check_content=True):
1033
"""See VersionedFiles.add_lines()."""
1034
path = self._mapper.map(key)
1035
version_id = key[-1]
1036
parents = [parent[-1] for parent in parents]
1037
vf = self._get_vf(path)
1040
return vf.add_lines_with_ghosts(version_id, parents, lines,
1041
parent_texts=parent_texts,
1042
left_matching_blocks=left_matching_blocks,
1043
nostore_sha=nostore_sha, random_id=random_id,
1044
check_content=check_content)
1045
except NotImplementedError:
1046
return vf.add_lines(version_id, parents, lines,
1047
parent_texts=parent_texts,
1048
left_matching_blocks=left_matching_blocks,
1049
nostore_sha=nostore_sha, random_id=random_id,
1050
check_content=check_content)
1051
except errors.NoSuchFile:
1052
# parent directory may be missing, try again.
1053
self._transport.mkdir(osutils.dirname(path))
1055
return vf.add_lines_with_ghosts(version_id, parents, lines,
1056
parent_texts=parent_texts,
1057
left_matching_blocks=left_matching_blocks,
1058
nostore_sha=nostore_sha, random_id=random_id,
1059
check_content=check_content)
1060
except NotImplementedError:
1061
return vf.add_lines(version_id, parents, lines,
1062
parent_texts=parent_texts,
1063
left_matching_blocks=left_matching_blocks,
1064
nostore_sha=nostore_sha, random_id=random_id,
1065
check_content=check_content)
1067
def annotate(self, key):
1068
"""Return a list of (version-key, line) tuples for the text of key.
1070
:raise RevisionNotPresent: If the key is not present.
1073
path = self._mapper.map(prefix)
1074
vf = self._get_vf(path)
1075
origins = vf.annotate(key[-1])
1077
for origin, line in origins:
1078
result.append((prefix + (origin,), line))
1081
def check(self, progress_bar=None):
1082
"""See VersionedFiles.check()."""
1083
for prefix, vf in self._iter_all_components():
1086
def get_parent_map(self, keys):
1087
"""Get a map of the parents of keys.
1089
:param keys: The keys to look up parents for.
1090
:return: A mapping from keys to parents. Absent keys are absent from
1093
prefixes = self._partition_keys(keys)
1095
for prefix, suffixes in prefixes.items():
1096
path = self._mapper.map(prefix)
1097
vf = self._get_vf(path)
1098
parent_map = vf.get_parent_map(suffixes)
1099
for key, parents in parent_map.items():
1100
result[prefix + (key,)] = tuple(
1101
prefix + (parent,) for parent in parents)
1104
def _get_vf(self, path):
1105
if not self._is_locked():
1106
raise errors.ObjectNotLocked(self)
1107
return self._file_factory(path, self._transport, create=True,
1108
get_scope=lambda:None)
1110
def _partition_keys(self, keys):
1111
"""Turn keys into a dict of prefix:suffix_list."""
1114
prefix_keys = result.setdefault(key[:-1], [])
1115
prefix_keys.append(key[-1])
1118
def _get_all_prefixes(self):
1119
# Identify all key prefixes.
1120
# XXX: A bit hacky, needs polish.
1121
if type(self._mapper) == ConstantMapper:
1122
paths = [self._mapper.map(())]
1126
for quoted_relpath in self._transport.iter_files_recursive():
1127
path, ext = os.path.splitext(quoted_relpath)
1129
paths = list(relpaths)
1130
prefixes = [self._mapper.unmap(path) for path in paths]
1131
return zip(paths, prefixes)
1133
def get_record_stream(self, keys, ordering, include_delta_closure):
1134
"""See VersionedFiles.get_record_stream()."""
1135
# Ordering will be taken care of by each partitioned store; group keys
1138
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1139
suffixes = [(suffix,) for suffix in suffixes]
1140
for record in vf.get_record_stream(suffixes, ordering,
1141
include_delta_closure):
1142
if record.parents is not None:
1143
record.parents = tuple(
1144
prefix + parent for parent in record.parents)
1145
record.key = prefix + record.key
1148
def _iter_keys_vf(self, keys):
1149
prefixes = self._partition_keys(keys)
1151
for prefix, suffixes in prefixes.items():
1152
path = self._mapper.map(prefix)
1153
vf = self._get_vf(path)
1154
yield prefix, suffixes, vf
1156
def get_sha1s(self, keys):
1157
"""See VersionedFiles.get_sha1s()."""
1159
for prefix,suffixes, vf in self._iter_keys_vf(keys):
1160
vf_sha1s = vf.get_sha1s(suffixes)
1161
for suffix, sha1 in vf_sha1s.iteritems():
1162
sha1s[prefix + (suffix,)] = sha1
1165
def insert_record_stream(self, stream):
1166
"""Insert a record stream into this container.
1168
:param stream: A stream of records to insert.
1170
:seealso VersionedFile.get_record_stream:
1172
for record in stream:
1173
prefix = record.key[:-1]
1174
key = record.key[-1:]
1175
if record.parents is not None:
1176
parents = [parent[-1:] for parent in record.parents]
1179
thunk_record = AdapterFactory(key, parents, record)
1180
path = self._mapper.map(prefix)
1181
# Note that this parses the file many times; we can do better but
1182
# as this only impacts weaves in terms of performance, it is
1184
vf = self._get_vf(path)
1185
vf.insert_record_stream([thunk_record])
1187
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1188
"""Iterate over the lines in the versioned files from keys.
1190
This may return lines from other keys. Each item the returned
1191
iterator yields is a tuple of a line and a text version that that line
1192
is present in (not introduced in).
1194
Ordering of results is in whatever order is most suitable for the
1195
underlying storage format.
1197
If a progress bar is supplied, it may be used to indicate progress.
1198
The caller is responsible for cleaning up progress bars (because this
1202
* Lines are normalised by the underlying store: they will all have \n
1204
* Lines are returned in arbitrary order.
1206
:return: An iterator over (line, key).
1208
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1209
for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1210
yield line, prefix + (version,)
1212
def _iter_all_components(self):
1213
for path, prefix in self._get_all_prefixes():
1214
yield prefix, self._get_vf(path)
1217
"""See VersionedFiles.keys()."""
1219
for prefix, vf in self._iter_all_components():
1220
for suffix in vf.versions():
1221
result.add(prefix + (suffix,))
1225
class _PlanMergeVersionedFile(VersionedFiles):
1226
"""A VersionedFile for uncommitted and committed texts.
1228
It is intended to allow merges to be planned with working tree texts.
1229
It implements only the small part of the VersionedFiles interface used by
1230
PlanMerge. It falls back to multiple versionedfiles for data not stored in
1231
_PlanMergeVersionedFile itself.
1233
:ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1234
queried for missing texts.
1237
def __init__(self, file_id):
1238
"""Create a _PlanMergeVersionedFile.
1240
:param file_id: Used with _PlanMerge code which is not yet fully
1241
tuple-keyspace aware.
1243
self._file_id = file_id
1244
# fallback locations
1245
self.fallback_versionedfiles = []
1246
# Parents for locally held keys.
1248
# line data for locally held keys.
1250
# key lookup providers
1251
self._providers = [DictParentsProvider(self._parents)]
1253
def plan_merge(self, ver_a, ver_b, base=None):
1254
"""See VersionedFile.plan_merge"""
1255
from bzrlib.merge import _PlanMerge
1257
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1258
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1259
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1260
return _PlanMerge._subtract_plans(old_plan, new_plan)
1262
def plan_lca_merge(self, ver_a, ver_b, base=None):
1263
from bzrlib.merge import _PlanLCAMerge
1265
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1268
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1269
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1271
def add_lines(self, key, parents, lines):
1272
"""See VersionedFiles.add_lines
1274
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1275
are permitted. Only reserved ids are permitted.
1277
if type(key) is not tuple:
1278
raise TypeError(key)
1279
if not revision.is_reserved_id(key[-1]):
1280
raise ValueError('Only reserved ids may be used')
1282
raise ValueError('Parents may not be None')
1284
raise ValueError('Lines may not be None')
1285
self._parents[key] = tuple(parents)
1286
self._lines[key] = lines
1288
def get_record_stream(self, keys, ordering, include_delta_closure):
1291
if key in self._lines:
1292
lines = self._lines[key]
1293
parents = self._parents[key]
1295
yield ChunkedContentFactory(key, parents, None, lines)
1296
for versionedfile in self.fallback_versionedfiles:
1297
for record in versionedfile.get_record_stream(
1298
pending, 'unordered', True):
1299
if record.storage_kind == 'absent':
1302
pending.remove(record.key)
1306
# report absent entries
1308
yield AbsentContentFactory(key)
1310
def get_parent_map(self, keys):
1311
"""See VersionedFiles.get_parent_map"""
1312
# We create a new provider because a fallback may have been added.
1313
# If we make fallbacks private we can update a stack list and avoid
1314
# object creation thrashing.
1317
if revision.NULL_REVISION in keys:
1318
keys.remove(revision.NULL_REVISION)
1319
result[revision.NULL_REVISION] = ()
1320
self._providers = self._providers[:1] + self.fallback_versionedfiles
1322
_StackedParentsProvider(self._providers).get_parent_map(keys))
1323
for key, parents in result.iteritems():
1325
result[key] = (revision.NULL_REVISION,)
1329
class PlanWeaveMerge(TextMerge):
1330
"""Weave merge that takes a plan as its input.
1332
This exists so that VersionedFile.plan_merge is implementable.
1333
Most callers will want to use WeaveMerge instead.
1336
def __init__(self, plan, a_marker=TextMerge.A_MARKER,
1337
b_marker=TextMerge.B_MARKER):
1338
TextMerge.__init__(self, a_marker, b_marker)
1341
def _merge_struct(self):
1346
def outstanding_struct():
1347
if not lines_a and not lines_b:
1349
elif ch_a and not ch_b:
1352
elif ch_b and not ch_a:
1354
elif lines_a == lines_b:
1357
yield (lines_a, lines_b)
1359
# We previously considered either 'unchanged' or 'killed-both' lines
1360
# to be possible places to resynchronize. However, assuming agreement
1361
# on killed-both lines may be too aggressive. -- mbp 20060324
1362
for state, line in self.plan:
1363
if state == 'unchanged':
1364
# resync and flush queued conflicts changes if any
1365
for struct in outstanding_struct():
1371
if state == 'unchanged':
1374
elif state == 'killed-a':
1376
lines_b.append(line)
1377
elif state == 'killed-b':
1379
lines_a.append(line)
1380
elif state == 'new-a':
1382
lines_a.append(line)
1383
elif state == 'new-b':
1385
lines_b.append(line)
1386
elif state == 'conflicted-a':
1388
lines_a.append(line)
1389
elif state == 'conflicted-b':
1391
lines_b.append(line)
1393
if state not in ('irrelevant', 'ghost-a', 'ghost-b',
1394
'killed-base', 'killed-both'):
1395
raise AssertionError(state)
1396
for struct in outstanding_struct():
1400
class WeaveMerge(PlanWeaveMerge):
1401
"""Weave merge that takes a VersionedFile and two versions as its input."""
1403
def __init__(self, versionedfile, ver_a, ver_b,
1404
a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1405
plan = versionedfile.plan_merge(ver_a, ver_b)
1406
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1409
class VirtualVersionedFiles(VersionedFiles):
1410
"""Dummy implementation for VersionedFiles that uses other functions for
1411
obtaining fulltexts and parent maps.
1413
This is always on the bottom of the stack and uses string keys
1414
(rather than tuples) internally.
1417
def __init__(self, get_parent_map, get_lines):
1418
"""Create a VirtualVersionedFiles.
1420
:param get_parent_map: Same signature as Repository.get_parent_map.
1421
:param get_lines: Should return lines for specified key or None if
1424
super(VirtualVersionedFiles, self).__init__()
1425
self._get_parent_map = get_parent_map
1426
self._get_lines = get_lines
1428
def check(self, progressbar=None):
1429
"""See VersionedFiles.check.
1431
:note: Always returns True for VirtualVersionedFiles.
1435
def add_mpdiffs(self, records):
1436
"""See VersionedFiles.mpdiffs.
1438
:note: Not implemented for VirtualVersionedFiles.
1440
raise NotImplementedError(self.add_mpdiffs)
1442
def get_parent_map(self, keys):
1443
"""See VersionedFiles.get_parent_map."""
1444
return dict([((k,), tuple([(p,) for p in v]))
1445
for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
1447
def get_sha1s(self, keys):
1448
"""See VersionedFiles.get_sha1s."""
1451
lines = self._get_lines(k)
1452
if lines is not None:
1453
if not isinstance(lines, list):
1454
raise AssertionError
1455
ret[(k,)] = osutils.sha_strings(lines)
1458
def get_record_stream(self, keys, ordering, include_delta_closure):
1459
"""See VersionedFiles.get_record_stream."""
1460
for (k,) in list(keys):
1461
lines = self._get_lines(k)
1462
if lines is not None:
1463
if not isinstance(lines, list):
1464
raise AssertionError
1465
yield ChunkedContentFactory((k,), None,
1466
sha1=osutils.sha_strings(lines),
1469
yield AbsentContentFactory((k,))
1471
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1472
"""See VersionedFile.iter_lines_added_or_present_in_versions()."""
1473
for i, (key,) in enumerate(keys):
1475
pb.update("iterating texts", i, len(keys))
1476
for l in self._get_lines(key):
1480
def network_bytes_to_kind_and_offset(network_bytes):
1481
"""Strip of a record kind from the front of network_bytes.
1483
:param network_bytes: The bytes of a record.
1484
:return: A tuple (storage_kind, offset_of_remaining_bytes)
1486
line_end = network_bytes.find('\n')
1487
storage_kind = network_bytes[:line_end]
1488
return storage_kind, line_end + 1
1491
class NetworkRecordStream(object):
1492
"""A record_stream which reconstitures a serialised stream."""
1494
def __init__(self, bytes_iterator):
1495
"""Create a NetworkRecordStream.
1497
:param bytes_iterator: An iterator of bytes. Each item in this
1498
iterator should have been obtained from a record_streams'
1499
record.get_bytes_as(record.storage_kind) call.
1501
self._bytes_iterator = bytes_iterator
1502
self._kind_factory = {'knit-ft-gz':knit.knit_network_to_record,
1503
'knit-delta-gz':knit.knit_network_to_record,
1504
'knit-annotated-ft-gz':knit.knit_network_to_record,
1505
'knit-annotated-delta-gz':knit.knit_network_to_record,
1506
'knit-delta-closure':knit.knit_delta_closure_to_records,
1507
'fulltext':fulltext_network_to_record,
1513
:return: An iterator as per VersionedFiles.get_record_stream().
1515
for bytes in self._bytes_iterator:
1516
storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
1517
for record in self._kind_factory[storage_kind](
1518
storage_kind, bytes, line_end):
1522
def fulltext_network_to_record(kind, bytes, line_end):
1523
"""Convert a network fulltext record to record."""
1524
meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
1525
record_meta = record_bytes[line_end+4:line_end+4+meta_len]
1526
key, parents = bencode.bdecode_as_tuple(record_meta)
1527
if parents == 'nil':
1529
fulltext = record_bytes[line_end+4+meta_len:]
1530
return FulltextContentFactory(key, parents, None, fulltext)
1533
def _length_prefix(bytes):
1534
return struct.pack('!L', len(bytes))
1537
def record_to_fulltext_bytes(self, record):
1538
if record.parents is None:
1541
parents = record.parents
1542
record_meta = bencode.bencode((record.key, parents))
1543
record_content = record.get_bytes_as('fulltext')
1544
return "fulltext\n%s%s%s" % (
1545
_length_prefix(record_meta), record_meta, record_content)
added
removed
bzrlib/versionedfile.py
