/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
4763.2.4 by John Arbash Meinel
merge bzr.2.1 in preparation for NEWS entry.
1
# Copyright (C) 2006-2010 Canonical Ltd
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
2
#
3
# Authors:
4
#   Johan Rydberg <jrydberg@gnu.org>
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
10
#
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
15
#
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
19
20
"""Versioned text file storage api."""
21
3350.8.2 by Robert Collins
stacked get_parent_map.
22
from copy import copy
3350.6.1 by Robert Collins
* New ``versionedfile.KeyMapper`` interface to abstract out the access to
23
from cStringIO import StringIO
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
24
import os
4022.1.6 by Robert Collins
Cherrypick and polish the RemoteSink for streaming push.
25
import struct
3350.6.1 by Robert Collins
* New ``versionedfile.KeyMapper`` interface to abstract out the access to
26
from zlib import adler32
27
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
28
from bzrlib.lazy_import import lazy_import
29
lazy_import(globals(), """
3224.5.20 by Andrew Bennetts
Remove or lazyify a couple more imports.
30
import urllib
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
31
32
from bzrlib import (
4454.3.65 by John Arbash Meinel
Tests that VF implementations support .get_annotator()
33
    annotate,
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
34
    errors,
4593.5.36 by John Arbash Meinel
a few more implementations of the interface.
35
    graph as _mod_graph,
3735.32.18 by John Arbash Meinel
We now support generating a network stream.
36
    groupcompress,
3830.3.12 by Martin Pool
Review cleanups: unify has_key impls, add missing_keys(), clean up exception blocks
37
    index,
4005.3.2 by Robert Collins
First passing NetworkRecordStream test - a fulltext from any record type which isn't a chunked or fulltext can be serialised and deserialised successfully.
38
    knit,
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
39
    osutils,
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
40
    multiparent,
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
41
    tsort,
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
42
    revision,
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
43
    ui,
44
    )
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
45
from bzrlib.graph import DictParentsProvider, Graph, StackedParentsProvider
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
46
from bzrlib.transport.memory import MemoryTransport
47
""")
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
48
from bzrlib.registry import Registry
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
49
from bzrlib.symbol_versioning import *
1551.6.7 by Aaron Bentley
Implemented two-way merge, refactored weave merge
50
from bzrlib.textmerge import TextMerge
2694.5.4 by Jelmer Vernooij
Move bzrlib.util.bencode to bzrlib._bencode_py.
51
from bzrlib import bencode
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
52
53
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
54
adapter_registry = Registry()
55
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
56
    'DeltaPlainToFullText')
57
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
58
    'FTPlainToFullText')
59
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
60
    'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
61
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
62
    'bzrlib.knit', 'DeltaAnnotatedToFullText')
63
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
64
    'bzrlib.knit', 'FTAnnotatedToUnannotated')
65
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
66
    'bzrlib.knit', 'FTAnnotatedToFullText')
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
67
# adapter_registry.register_lazy(('knit-annotated-ft-gz', 'chunked'),
68
#     'bzrlib.knit', 'FTAnnotatedToChunked')
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
69
70
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
71
class ContentFactory(object):
72
    """Abstract interface for insertion and retrieval from a VersionedFile.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
73
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
74
    :ivar sha1: None, or the sha1 of the content fulltext.
75
    :ivar storage_kind: The native storage kind of this factory. One of
76
        'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
77
        'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
78
        'knit-annotated-delta-gz', 'knit-ft-gz', 'knit-delta-gz'.
79
    :ivar key: The key of this content. Each key is a tuple with a single
80
        string in it.
81
    :ivar parents: A tuple of parent keys for self.key. If the object has
82
        no parent information, None (as opposed to () for an empty list of
83
        parents).
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
84
    """
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
85
86
    def __init__(self):
87
        """Create a ContentFactory."""
88
        self.sha1 = None
89
        self.storage_kind = None
90
        self.key = None
91
        self.parents = None
92
93
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
94
class ChunkedContentFactory(ContentFactory):
95
    """Static data content factory.
96
97
    This takes a 'chunked' list of strings. The only requirement on 'chunked' is
98
    that ''.join(lines) becomes a valid fulltext. A tuple of a single string
99
    satisfies this, as does a list of lines.
100
101
    :ivar sha1: None, or the sha1 of the content fulltext.
102
    :ivar storage_kind: The native storage kind of this factory. Always
3890.2.2 by John Arbash Meinel
Change the signature to report the storage kind as 'chunked'
103
        'chunked'
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
104
    :ivar key: The key of this content. Each key is a tuple with a single
105
        string in it.
106
    :ivar parents: A tuple of parent keys for self.key. If the object has
107
        no parent information, None (as opposed to () for an empty list of
108
        parents).
109
     """
110
111
    def __init__(self, key, parents, sha1, chunks):
112
        """Create a ContentFactory."""
113
        self.sha1 = sha1
3890.2.2 by John Arbash Meinel
Change the signature to report the storage kind as 'chunked'
114
        self.storage_kind = 'chunked'
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
115
        self.key = key
116
        self.parents = parents
117
        self._chunks = chunks
118
119
    def get_bytes_as(self, storage_kind):
120
        if storage_kind == 'chunked':
121
            return self._chunks
122
        elif storage_kind == 'fulltext':
123
            return ''.join(self._chunks)
124
        raise errors.UnavailableRepresentation(self.key, storage_kind,
125
            self.storage_kind)
126
127
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
128
class FulltextContentFactory(ContentFactory):
129
    """Static data content factory.
130
131
    This takes a fulltext when created and just returns that during
132
    get_bytes_as('fulltext').
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
133
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
134
    :ivar sha1: None, or the sha1 of the content fulltext.
135
    :ivar storage_kind: The native storage kind of this factory. Always
136
        'fulltext'.
137
    :ivar key: The key of this content. Each key is a tuple with a single
138
        string in it.
139
    :ivar parents: A tuple of parent keys for self.key. If the object has
140
        no parent information, None (as opposed to () for an empty list of
141
        parents).
142
     """
143
144
    def __init__(self, key, parents, sha1, text):
145
        """Create a ContentFactory."""
146
        self.sha1 = sha1
147
        self.storage_kind = 'fulltext'
148
        self.key = key
149
        self.parents = parents
150
        self._text = text
151
152
    def get_bytes_as(self, storage_kind):
153
        if storage_kind == self.storage_kind:
154
            return self._text
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
155
        elif storage_kind == 'chunked':
3976.2.1 by Robert Collins
Use a list not a tuple for chunks returned from FullTextContentFactory objects, because otherwise code tries to assign to tuples.
156
            return [self._text]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
157
        raise errors.UnavailableRepresentation(self.key, storage_kind,
158
            self.storage_kind)
159
160
161
class AbsentContentFactory(ContentFactory):
3350.3.12 by Robert Collins
Generate streams with absent records.
162
    """A placeholder content factory for unavailable texts.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
163
3350.3.12 by Robert Collins
Generate streams with absent records.
164
    :ivar sha1: None.
165
    :ivar storage_kind: 'absent'.
166
    :ivar key: The key of this content. Each key is a tuple with a single
167
        string in it.
168
    :ivar parents: None.
169
    """
170
171
    def __init__(self, key):
172
        """Create a ContentFactory."""
173
        self.sha1 = None
174
        self.storage_kind = 'absent'
175
        self.key = key
176
        self.parents = None
177
4537.2.1 by John Arbash Meinel
Add AbsentContentFactory.get_bytes_as()
178
    def get_bytes_as(self, storage_kind):
179
        raise ValueError('A request was made for key: %s, but that'
180
                         ' content is not available, and the calling'
181
                         ' code does not handle if it is missing.'
182
                         % (self.key,))
183
3350.3.12 by Robert Collins
Generate streams with absent records.
184
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
185
class AdapterFactory(ContentFactory):
186
    """A content factory to adapt between key prefix's."""
187
188
    def __init__(self, key, parents, adapted):
189
        """Create an adapter factory instance."""
190
        self.key = key
191
        self.parents = parents
192
        self._adapted = adapted
193
194
    def __getattr__(self, attr):
195
        """Return a member from the adapted object."""
196
        if attr in ('key', 'parents'):
197
            return self.__dict__[attr]
198
        else:
199
            return getattr(self._adapted, attr)
200
201
3350.3.14 by Robert Collins
Deprecate VersionedFile.join.
202
def filter_absent(record_stream):
203
    """Adapt a record stream to remove absent records."""
204
    for record in record_stream:
205
        if record.storage_kind != 'absent':
206
            yield record
207
208
5374.2.2 by John Arbash Meinel
Create a multi-parent diff generator class.
209
class _MPDiffGenerator(object):
210
    """Pull out the functionality for generating mp_diffs."""
211
212
    def __init__(self, vf, keys):
213
        self.vf = vf
214
        # This is the order the keys were requested in
215
        self.ordered_keys = tuple(keys)
216
        # keys + their parents, what we need to compute the diffs
217
        self.needed_keys = ()
218
        # Map from key: mp_diff
219
        self.diffs = {}
220
        # Map from key: parents_needed (may have ghosts)
221
        self.parent_map = {}
222
        # Parents that aren't present
223
        self.ghost_parents = ()
224
        # Map from parent_key => number of children for this text
225
        self.refcounts = {}
226
        # Content chunks that are cached while we still need them
227
        self.chunks = {}
228
229
    def _find_needed_keys(self):
230
        """Find the set of keys we need to request.
231
232
        This includes all the original keys passed in, and the non-ghost
233
        parents of those keys.
234
235
        :return: (needed_keys, refcounts)
236
            needed_keys is the set of all texts we need to extract
237
            refcounts is a dict of {key: num_children} letting us know when we
238
                no longer need to cache a given parent text
239
        """
240
        # All the keys and their parents
241
        needed_keys = set(self.ordered_keys)
242
        parent_map = self.vf.get_parent_map(needed_keys)
243
        self.parent_map = parent_map
244
        # TODO: Should we be using a different construct here? I think this
245
        #       uses difference_update internally, and we expect the result to
246
        #       be tiny
247
        missing_keys = needed_keys.difference(parent_map)
248
        if missing_keys:
249
            raise errors.RevisionNotPresent(list(missing_keys)[0], self.vf)
250
        # Parents that might be missing. They are allowed to be ghosts, but we
251
        # should check for them
252
        refcounts = {}
253
        setdefault = refcounts.setdefault
254
        maybe_ghosts = set()
255
        for child_key, parent_keys in parent_map.iteritems():
256
            if not parent_keys:
257
                # Ghost? We should never get here
258
                continue
259
            maybe_ghosts.update(p for p in parent_keys if p not in needed_keys)
260
            needed_keys.update(parent_keys)
261
            for p in parent_keys:
262
                refcounts[p] = setdefault(p, 0) + 1
263
        # Remove any parents that are actually ghosts
264
        self.ghost_parents = maybe_ghosts.difference(
265
                                self.vf.get_parent_map(maybe_ghosts))
266
        needed_keys.difference_update(self.ghost_parents)
267
        self.needed_keys = needed_keys
268
        self.refcounts = refcounts
269
        return needed_keys, refcounts
270
271
    def _compute_diff(self, key, parent_lines, lines):
272
        """Compute a single mp_diff, and store it in self._diffs"""
273
        if len(parent_lines) > 0:
274
            # XXX: _extract_blocks is not usefully defined anywhere...
275
            #      It was meant to extract the left-parent diff without
276
            #      having to recompute it for Knit content (pack-0.92,
277
            #      etc). That seems to have regressed somewhere
278
            left_parent_blocks = self.vf._extract_blocks(key,
279
                parent_lines[0], lines)
280
        else:
281
            left_parent_blocks = None
282
        diff = multiparent.MultiParent.from_lines(lines,
283
                    parent_lines, left_parent_blocks)
284
        self.diffs[key] = diff
285
286
    def _process_one_record(self, record):
287
        this_chunks = record.get_bytes_as('chunked')
288
        if record.key in self.parent_map:
289
            # This record should be ready to diff, since we requested
290
            # content in 'topological' order
291
            parent_keys = self.parent_map.pop(record.key)
5374.2.3 by John Arbash Meinel
Replace the generic VersionedFiles.make_mpdiffs interface.
292
            # If a VersionedFile claims 'no-graph' support, then it may return
293
            # None for any parent request, so we replace it with an empty tuple
294
            if parent_keys is None:
295
                parent_keys = ()
5374.2.2 by John Arbash Meinel
Create a multi-parent diff generator class.
296
            parent_lines = []
297
            for p in parent_keys:
298
                # Alternatively we could check p not in self.needed_keys, but
299
                # ghost_parents should be tiny versus huge
300
                if p in self.ghost_parents:
301
                    continue
302
                refcount = self.refcounts[p]
303
                if refcount == 1: # Last child reference
304
                    self.refcounts.pop(p)
305
                    parent_chunks = self.chunks.pop(p)
306
                else:
307
                    self.refcounts[p] = refcount - 1
308
                    parent_chunks = self.chunks[p]
309
                # TODO: Should we cache the line form? We did the
310
                #       computation to get it, but storing it this way will
311
                #       be less memory efficient...
312
                parent_lines.append(osutils.chunks_to_lines(parent_chunks))
313
            lines = osutils.chunks_to_lines(this_chunks)
314
            # TODO: Should we be caching lines instead of chunks?
315
            #       Higher-memory, but avoids double extracting.
316
            #       If we have good topological sorting, we shouldn't have
317
            #       much pending stuff cached...
318
            ## this_chunks = lines
319
            self._compute_diff(record.key, parent_lines, lines)
320
        # Is this content required for any more children?
321
        if record.key in self.refcounts:
322
            self.chunks[record.key] = this_chunks
323
324
    def _extract_diffs(self):
325
        needed_keys, refcounts = self._find_needed_keys()
326
        for record in self.vf.get_record_stream(needed_keys,
327
                                                'topological', True):
328
            if record.storage_kind == 'absent':
329
                raise errors.RevisionNotPresent(record.key, self.vf)
330
            self._process_one_record(record)
331
        # At this point, we should have *no* remaining content
332
        assert not self.parent_map
333
        assert set(self.refcounts) == self.ghost_parents
334
        
335
    def compute_diffs(self):
336
        self._extract_diffs()
337
        dpop = self.diffs.pop
338
        return [dpop(k) for k in self.ordered_keys]
339
340
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
341
class VersionedFile(object):
342
    """Versioned text file storage.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
343
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
344
    A versioned file manages versions of line-based text files,
345
    keeping track of the originating version for each line.
346
347
    To clients the "lines" of the file are represented as a list of
348
    strings. These strings will typically have terminal newline
349
    characters, but this is not required.  In particular files commonly
350
    do not have a newline at the end of the file.
351
352
    Texts are identified by a version-id string.
353
    """
354
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
355
    @staticmethod
2229.2.3 by Aaron Bentley
change reserved_id to is_reserved_id, add check_not_reserved for DRY
356
    def check_not_reserved_id(version_id):
357
        revision.check_not_reserved_id(version_id)
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
358
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
359
    def copy_to(self, name, transport):
360
        """Copy this versioned file to name on transport."""
361
        raise NotImplementedError(self.copy_to)
1863.1.1 by John Arbash Meinel
Allow Versioned files to do caching if explicitly asked, and implement for Knit
362
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
363
    def get_record_stream(self, versions, ordering, include_delta_closure):
364
        """Get a stream of records for versions.
365
366
        :param versions: The versions to include. Each version is a tuple
367
            (version,).
368
        :param ordering: Either 'unordered' or 'topological'. A topologically
369
            sorted stream has compression parents strictly before their
370
            children.
371
        :param include_delta_closure: If True then the closure across any
3350.3.22 by Robert Collins
Review feedback.
372
            compression parents will be included (in the data content of the
373
            stream, not in the emitted records). This guarantees that
374
            'fulltext' can be used successfully on every record.
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
375
        :return: An iterator of ContentFactory objects, each of which is only
376
            valid until the iterator is advanced.
377
        """
378
        raise NotImplementedError(self.get_record_stream)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
379
380
    def has_version(self, version_id):
381
        """Returns whether version is present."""
382
        raise NotImplementedError(self.has_version)
383
5195.3.26 by Parth Malwankar
reverted changes done to insert_record_stream API
384
    def insert_record_stream(self, stream):
3350.3.8 by Robert Collins
Basic stream insertion, no fast path yet for knit to knit.
385
        """Insert a record stream into this versioned file.
386
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
387
        :param stream: A stream of records to insert.
3350.3.8 by Robert Collins
Basic stream insertion, no fast path yet for knit to knit.
388
        :return: None
389
        :seealso VersionedFile.get_record_stream:
390
        """
391
        raise NotImplementedError
392
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
393
    def add_lines(self, version_id, parents, lines, parent_texts=None,
2805.6.7 by Robert Collins
Review feedback.
394
        left_matching_blocks=None, nostore_sha=None, random_id=False,
395
        check_content=True):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
396
        """Add a single text on top of the versioned file.
397
398
        Must raise RevisionAlreadyPresent if the new version is
399
        already present in file history.
400
401
        Must raise RevisionNotPresent if any of the given parents are
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
402
        not present in file history.
2805.6.3 by Robert Collins
* The ``VersionedFile`` interface no longer protects against misuse when
403
404
        :param lines: A list of lines. Each line must be a bytestring. And all
405
            of them except the last must be terminated with \n and contain no
406
            other \n's. The last line may either contain no \n's or a single
407
            terminated \n. If the lines list does meet this constraint the add
408
            routine may error or may succeed - but you will be unable to read
409
            the data back accurately. (Checking the lines have been split
2805.6.7 by Robert Collins
Review feedback.
410
            correctly is expensive and extremely unlikely to catch bugs so it
411
            is not done at runtime unless check_content is True.)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
412
        :param parent_texts: An optional dictionary containing the opaque
2805.6.3 by Robert Collins
* The ``VersionedFile`` interface no longer protects against misuse when
413
            representations of some or all of the parents of version_id to
414
            allow delta optimisations.  VERY IMPORTANT: the texts must be those
415
            returned by add_lines or data corruption can be caused.
2520.4.148 by Aaron Bentley
Updates from review
416
        :param left_matching_blocks: a hint about which areas are common
417
            between the text and its left-hand-parent.  The format is
418
            the SequenceMatcher.get_matching_blocks format.
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
419
        :param nostore_sha: Raise ExistingContent and do not add the lines to
420
            the versioned file if the digest of the lines matches this.
2805.6.4 by Robert Collins
Don't check for existing versions when adding texts with random revision ids.
421
        :param random_id: If True a random id has been selected rather than
422
            an id determined by some deterministic process such as a converter
423
            from a foreign VCS. When True the backend may choose not to check
424
            for uniqueness of the resulting key within the versioned file, so
425
            this should only be done when the result is expected to be unique
426
            anyway.
2805.6.7 by Robert Collins
Review feedback.
427
        :param check_content: If True, the lines supplied are verified to be
428
            bytestrings that are correctly formed lines.
2776.1.1 by Robert Collins
* The ``add_lines`` methods on ``VersionedFile`` implementations has changed
429
        :return: The text sha1, the number of bytes in the text, and an opaque
430
                 representation of the inserted version which can be provided
431
                 back to future add_lines calls in the parent_texts dictionary.
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
432
        """
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
433
        self._check_write_ok()
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
434
        return self._add_lines(version_id, parents, lines, parent_texts,
2805.6.7 by Robert Collins
Review feedback.
435
            left_matching_blocks, nostore_sha, random_id, check_content)
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
436
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
437
    def _add_lines(self, version_id, parents, lines, parent_texts,
2805.6.7 by Robert Collins
Review feedback.
438
        left_matching_blocks, nostore_sha, random_id, check_content):
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
439
        """Helper to do the class specific add_lines."""
1563.2.4 by Robert Collins
First cut at including the knit implementation of versioned_file.
440
        raise NotImplementedError(self.add_lines)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
441
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
442
    def add_lines_with_ghosts(self, version_id, parents, lines,
2805.6.7 by Robert Collins
Review feedback.
443
        parent_texts=None, nostore_sha=None, random_id=False,
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
444
        check_content=True, left_matching_blocks=None):
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
445
        """Add lines to the versioned file, allowing ghosts to be present.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
446
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
447
        This takes the same parameters as add_lines and returns the same.
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
448
        """
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
449
        self._check_write_ok()
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
450
        return self._add_lines_with_ghosts(version_id, parents, lines,
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
451
            parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
452
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
453
    def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
454
        nostore_sha, random_id, check_content, left_matching_blocks):
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
455
        """Helper to do class specific add_lines_with_ghosts."""
1594.2.8 by Robert Collins
add ghost aware apis to knits.
456
        raise NotImplementedError(self.add_lines_with_ghosts)
457
1563.2.19 by Robert Collins
stub out a check for knits.
458
    def check(self, progress_bar=None):
459
        """Check the versioned file for integrity."""
460
        raise NotImplementedError(self.check)
461
1666.1.6 by Robert Collins
Make knit the default format.
462
    def _check_lines_not_unicode(self, lines):
463
        """Check that lines being added to a versioned file are not unicode."""
464
        for line in lines:
465
            if line.__class__ is not str:
466
                raise errors.BzrBadParameterUnicode("lines")
467
468
    def _check_lines_are_lines(self, lines):
469
        """Check that the lines really are full lines without inline EOL."""
470
        for line in lines:
471
            if '\n' in line[:-1]:
472
                raise errors.BzrBadParameterContainsNewline("lines")
473
2535.3.1 by Andrew Bennetts
Add get_format_signature to VersionedFile
474
    def get_format_signature(self):
475
        """Get a text description of the data encoding in this file.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
476
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
477
        :since: 0.90
2535.3.1 by Andrew Bennetts
Add get_format_signature to VersionedFile
478
        """
479
        raise NotImplementedError(self.get_format_signature)
480
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
481
    def make_mpdiffs(self, version_ids):
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
482
        """Create multiparent diffs for specified versions."""
5374.2.3 by John Arbash Meinel
Replace the generic VersionedFiles.make_mpdiffs interface.
483
        # XXX: Can't use _MPDiffGenerator just yet. This is because version_ids
484
        #      is a list of strings, not keys. And while self.get_record_stream
485
        #      is supported, it takes *keys*, while self.get_parent_map() takes
486
        #      strings... *sigh*
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
487
        knit_versions = set()
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
488
        knit_versions.update(version_ids)
489
        parent_map = self.get_parent_map(version_ids)
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
490
        for version_id in version_ids:
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
491
            try:
492
                knit_versions.update(parent_map[version_id])
493
            except KeyError:
3453.3.1 by Daniel Fischer
Raise the right exception in make_mpdiffs (bug #235687)
494
                raise errors.RevisionNotPresent(version_id, self)
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
495
        # We need to filter out ghosts, because we can't diff against them.
496
        knit_versions = set(self.get_parent_map(knit_versions).keys())
2520.4.90 by Aaron Bentley
Handle \r terminated lines in Weaves properly
497
        lines = dict(zip(knit_versions,
498
            self._get_lf_split_line_list(knit_versions)))
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
499
        diffs = []
500
        for version_id in version_ids:
501
            target = lines[version_id]
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
502
            try:
503
                parents = [lines[p] for p in parent_map[version_id] if p in
504
                    knit_versions]
505
            except KeyError:
3453.3.2 by John Arbash Meinel
Add a test case for the first loop, unable to find a way to trigger the second loop
506
                # I don't know how this could ever trigger.
507
                # parent_map[version_id] was already triggered in the previous
508
                # for loop, and lines[p] has the 'if p in knit_versions' check,
509
                # so we again won't have a KeyError.
3453.3.1 by Daniel Fischer
Raise the right exception in make_mpdiffs (bug #235687)
510
                raise errors.RevisionNotPresent(version_id, self)
2520.4.48 by Aaron Bentley
Support getting blocks from knit deltas with no final EOL
511
            if len(parents) > 0:
512
                left_parent_blocks = self._extract_blocks(version_id,
513
                                                          parents[0], target)
514
            else:
515
                left_parent_blocks = None
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
516
            diffs.append(multiparent.MultiParent.from_lines(target, parents,
517
                         left_parent_blocks))
518
        return diffs
519
2520.4.48 by Aaron Bentley
Support getting blocks from knit deltas with no final EOL
520
    def _extract_blocks(self, version_id, source, target):
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
521
        return None
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
522
2520.4.61 by Aaron Bentley
Do bulk insertion of records
523
    def add_mpdiffs(self, records):
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
524
        """Add mpdiffs to this VersionedFile.
2520.4.126 by Aaron Bentley
Add more docs
525
526
        Records should be iterables of version, parents, expected_sha1,
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
527
        mpdiff. mpdiff should be a MultiParent instance.
2520.4.126 by Aaron Bentley
Add more docs
528
        """
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
529
        # Does this need to call self._check_write_ok()? (IanC 20070919)
2520.4.61 by Aaron Bentley
Do bulk insertion of records
530
        vf_parents = {}
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
531
        mpvf = multiparent.MultiMemoryVersionedFile()
532
        versions = []
533
        for version, parent_ids, expected_sha1, mpdiff in records:
534
            versions.append(version)
535
            mpvf.add_diff(mpdiff, version, parent_ids)
536
        needed_parents = set()
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
537
        for version, parent_ids, expected_sha1, mpdiff in records:
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
538
            needed_parents.update(p for p in parent_ids
539
                                  if not mpvf.has_version(p))
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
540
        present_parents = set(self.get_parent_map(needed_parents).keys())
541
        for parent_id, lines in zip(present_parents,
542
                                 self._get_lf_split_line_list(present_parents)):
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
543
            mpvf.add_version(lines, parent_id, [])
544
        for (version, parent_ids, expected_sha1, mpdiff), lines in\
545
            zip(records, mpvf.get_line_list(versions)):
546
            if len(parent_ids) == 1:
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
547
                left_matching_blocks = list(mpdiff.get_matching_blocks(0,
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
548
                    mpvf.get_diff(parent_ids[0]).num_lines()))
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
549
            else:
550
                left_matching_blocks = None
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
551
            try:
552
                _, _, version_text = self.add_lines_with_ghosts(version,
553
                    parent_ids, lines, vf_parents,
554
                    left_matching_blocks=left_matching_blocks)
555
            except NotImplementedError:
556
                # The vf can't handle ghosts, so add lines normally, which will
557
                # (reasonably) fail if there are ghosts in the data.
558
                _, _, version_text = self.add_lines(version,
559
                    parent_ids, lines, vf_parents,
560
                    left_matching_blocks=left_matching_blocks)
2520.4.61 by Aaron Bentley
Do bulk insertion of records
561
            vf_parents[version] = version_text
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
562
        sha1s = self.get_sha1s(versions)
563
        for version, parent_ids, expected_sha1, mpdiff in records:
564
            if expected_sha1 != sha1s[version]:
2520.4.71 by Aaron Bentley
Update test to accept VersionedFileInvalidChecksum instead of TestamentMismatch
565
                raise errors.VersionedFileInvalidChecksum(version)
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
566
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
567
    def get_text(self, version_id):
568
        """Return version contents as a text string.
569
570
        Raises RevisionNotPresent if version is not present in
571
        file history.
572
        """
573
        return ''.join(self.get_lines(version_id))
574
    get_string = get_text
575
1756.2.1 by Aaron Bentley
Implement get_texts
576
    def get_texts(self, version_ids):
577
        """Return the texts of listed versions as a list of strings.
578
579
        Raises RevisionNotPresent if version is not present in
580
        file history.
581
        """
582
        return [''.join(self.get_lines(v)) for v in version_ids]
583
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
584
    def get_lines(self, version_id):
585
        """Return version contents as a sequence of lines.
586
587
        Raises RevisionNotPresent if version is not present in
588
        file history.
589
        """
590
        raise NotImplementedError(self.get_lines)
591
2520.4.90 by Aaron Bentley
Handle \r terminated lines in Weaves properly
592
    def _get_lf_split_line_list(self, version_ids):
593
        return [StringIO(t).readlines() for t in self.get_texts(version_ids)]
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
594
2530.1.1 by Aaron Bentley
Make topological sorting optional for get_ancestry
595
    def get_ancestry(self, version_ids, topo_sorted=True):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
596
        """Return a list of all ancestors of given version(s). This
597
        will not include the null revision.
598
2490.2.32 by Aaron Bentley
Merge of not-sorting-ancestry branch
599
        This list will not be topologically sorted if topo_sorted=False is
600
        passed.
2530.1.1 by Aaron Bentley
Make topological sorting optional for get_ancestry
601
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
602
        Must raise RevisionNotPresent if any of the given versions are
603
        not present in file history."""
604
        if isinstance(version_ids, basestring):
605
            version_ids = [version_ids]
606
        raise NotImplementedError(self.get_ancestry)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
607
1594.2.8 by Robert Collins
add ghost aware apis to knits.
608
    def get_ancestry_with_ghosts(self, version_ids):
609
        """Return a list of all ancestors of given version(s). This
610
        will not include the null revision.
611
612
        Must raise RevisionNotPresent if any of the given versions are
613
        not present in file history.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
614
1594.2.8 by Robert Collins
add ghost aware apis to knits.
615
        Ghosts that are known about will be included in ancestry list,
616
        but are not explicitly marked.
617
        """
618
        raise NotImplementedError(self.get_ancestry_with_ghosts)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
619
3287.5.1 by Robert Collins
Add VersionedFile.get_parent_map.
620
    def get_parent_map(self, version_ids):
621
        """Get a map of the parents of version_ids.
622
623
        :param version_ids: The version ids to look up parents for.
624
        :return: A mapping from version id to parents.
625
        """
626
        raise NotImplementedError(self.get_parent_map)
627
1594.2.8 by Robert Collins
add ghost aware apis to knits.
628
    def get_parents_with_ghosts(self, version_id):
629
        """Return version names for parents of version_id.
630
631
        Will raise RevisionNotPresent if version_id is not present
632
        in the history.
633
634
        Ghosts that are known about will be included in the parent list,
635
        but are not explicitly marked.
636
        """
3287.5.1 by Robert Collins
Add VersionedFile.get_parent_map.
637
        try:
638
            return list(self.get_parent_map([version_id])[version_id])
639
        except KeyError:
640
            raise errors.RevisionNotPresent(version_id, self)
1594.2.8 by Robert Collins
add ghost aware apis to knits.
641
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
642
    def annotate(self, version_id):
3316.2.13 by Robert Collins
* ``VersionedFile.annotate_iter`` is deprecated. While in principal this
643
        """Return a list of (version-id, line) tuples for version_id.
644
645
        :raise RevisionNotPresent: If the given version is
646
        not present in file history.
647
        """
648
        raise NotImplementedError(self.annotate)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
649
2975.3.2 by Robert Collins
Review feedback - document the API change and improve readability in pack's _do_copy_nodes.
650
    def iter_lines_added_or_present_in_versions(self, version_ids=None,
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
651
                                                pb=None):
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
652
        """Iterate over the lines in the versioned file from version_ids.
653
2975.3.2 by Robert Collins
Review feedback - document the API change and improve readability in pack's _do_copy_nodes.
654
        This may return lines from other versions. Each item the returned
655
        iterator yields is a tuple of a line and a text version that that line
656
        is present in (not introduced in).
657
658
        Ordering of results is in whatever order is most suitable for the
659
        underlying storage format.
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
660
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
661
        If a progress bar is supplied, it may be used to indicate progress.
662
        The caller is responsible for cleaning up progress bars (because this
663
        is an iterator).
664
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
665
        NOTES: Lines are normalised: they will all have \n terminators.
666
               Lines are returned in arbitrary order.
2975.3.2 by Robert Collins
Review feedback - document the API change and improve readability in pack's _do_copy_nodes.
667
668
        :return: An iterator over (line, version_id).
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
669
        """
670
        raise NotImplementedError(self.iter_lines_added_or_present_in_versions)
671
1551.6.15 by Aaron Bentley
Moved plan_merge into Weave
672
    def plan_merge(self, ver_a, ver_b):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
673
        """Return pseudo-annotation indicating how the two versions merge.
674
675
        This is computed between versions a and b and their common
676
        base.
677
678
        Weave lines present in none of them are skipped entirely.
1664.2.2 by Aaron Bentley
Added legend for plan-merge output
679
680
        Legend:
681
        killed-base Dead in base revision
682
        killed-both Killed in each revision
683
        killed-a    Killed in a
684
        killed-b    Killed in b
685
        unchanged   Alive in both a and b (possibly created in both)
686
        new-a       Created in a
687
        new-b       Created in b
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
688
        ghost-a     Killed in a, unborn in b
1664.2.5 by Aaron Bentley
Update plan-merge legend
689
        ghost-b     Killed in b, unborn in a
1664.2.2 by Aaron Bentley
Added legend for plan-merge output
690
        irrelevant  Not in either revision
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
691
        """
1551.6.15 by Aaron Bentley
Moved plan_merge into Weave
692
        raise NotImplementedError(VersionedFile.plan_merge)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
693
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
694
    def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
1551.6.14 by Aaron Bentley
Tweaks from merge review
695
                    b_marker=TextMerge.B_MARKER):
1551.6.12 by Aaron Bentley
Indicate conflicts from merge_lines, insead of guessing
696
        return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
697
1664.2.7 by Aaron Bentley
Merge bzr.dev
698
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
699
class RecordingVersionedFilesDecorator(object):
700
    """A minimal versioned files that records calls made on it.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
701
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
702
    Only enough methods have been added to support tests using it to date.
703
704
    :ivar calls: A list of the calls made; can be reset at any time by
705
        assigning [] to it.
706
    """
707
708
    def __init__(self, backing_vf):
3871.4.1 by John Arbash Meinel
Add a VFDecorator that can yield records in a specified order
709
        """Create a RecordingVersionedFilesDecorator decorating backing_vf.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
710
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
711
        :param backing_vf: The versioned file to answer all methods.
712
        """
713
        self._backing_vf = backing_vf
714
        self.calls = []
715
3350.8.2 by Robert Collins
stacked get_parent_map.
716
    def add_lines(self, key, parents, lines, parent_texts=None,
717
        left_matching_blocks=None, nostore_sha=None, random_id=False,
718
        check_content=True):
719
        self.calls.append(("add_lines", key, parents, lines, parent_texts,
720
            left_matching_blocks, nostore_sha, random_id, check_content))
721
        return self._backing_vf.add_lines(key, parents, lines, parent_texts,
722
            left_matching_blocks, nostore_sha, random_id, check_content)
723
3517.4.19 by Martin Pool
Update test for knit.check() to expect it to recurse into fallback vfs
724
    def check(self):
725
        self._backing_vf.check()
726
3350.8.2 by Robert Collins
stacked get_parent_map.
727
    def get_parent_map(self, keys):
728
        self.calls.append(("get_parent_map", copy(keys)))
729
        return self._backing_vf.get_parent_map(keys)
730
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
731
    def get_record_stream(self, keys, sort_order, include_delta_closure):
3350.8.7 by Robert Collins
get_record_stream for fulltexts working (but note extreme memory use!).
732
        self.calls.append(("get_record_stream", list(keys), sort_order,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
733
            include_delta_closure))
734
        return self._backing_vf.get_record_stream(keys, sort_order,
735
            include_delta_closure)
736
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
737
    def get_sha1s(self, keys):
738
        self.calls.append(("get_sha1s", copy(keys)))
739
        return self._backing_vf.get_sha1s(keys)
740
3350.8.5 by Robert Collins
Iter_lines_added_or_present_in_keys stacks.
741
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
742
        self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
3350.8.14 by Robert Collins
Review feedback.
743
        return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
3350.8.5 by Robert Collins
Iter_lines_added_or_present_in_keys stacks.
744
3350.8.4 by Robert Collins
Vf.keys() stacking support.
745
    def keys(self):
746
        self.calls.append(("keys",))
747
        return self._backing_vf.keys()
748
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
749
3871.4.1 by John Arbash Meinel
Add a VFDecorator that can yield records in a specified order
750
class OrderingVersionedFilesDecorator(RecordingVersionedFilesDecorator):
751
    """A VF that records calls, and returns keys in specific order.
752
753
    :ivar calls: A list of the calls made; can be reset at any time by
754
        assigning [] to it.
755
    """
756
757
    def __init__(self, backing_vf, key_priority):
758
        """Create a RecordingVersionedFilesDecorator decorating backing_vf.
759
760
        :param backing_vf: The versioned file to answer all methods.
761
        :param key_priority: A dictionary defining what order keys should be
762
            returned from an 'unordered' get_record_stream request.
763
            Keys with lower priority are returned first, keys not present in
764
            the map get an implicit priority of 0, and are returned in
765
            lexicographical order.
766
        """
767
        RecordingVersionedFilesDecorator.__init__(self, backing_vf)
768
        self._key_priority = key_priority
769
770
    def get_record_stream(self, keys, sort_order, include_delta_closure):
771
        self.calls.append(("get_record_stream", list(keys), sort_order,
772
            include_delta_closure))
773
        if sort_order == 'unordered':
774
            def sort_key(key):
775
                return (self._key_priority.get(key, 0), key)
776
            # Use a defined order by asking for the keys one-by-one from the
777
            # backing_vf
778
            for key in sorted(keys, key=sort_key):
779
                for record in self._backing_vf.get_record_stream([key],
780
                                'unordered', include_delta_closure):
781
                    yield record
782
        else:
783
            for record in self._backing_vf.get_record_stream(keys, sort_order,
784
                            include_delta_closure):
785
                yield record
786
787
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
788
class KeyMapper(object):
3350.6.10 by Martin Pool
VersionedFiles review cleanups
789
    """KeyMappers map between keys and underlying partitioned storage."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
790
791
    def map(self, key):
792
        """Map key to an underlying storage identifier.
793
794
        :param key: A key tuple e.g. ('file-id', 'revision-id').
795
        :return: An underlying storage identifier, specific to the partitioning
796
            mechanism.
797
        """
798
        raise NotImplementedError(self.map)
799
800
    def unmap(self, partition_id):
801
        """Map a partitioned storage id back to a key prefix.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
802
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
803
        :param partition_id: The underlying partition id.
3350.6.10 by Martin Pool
VersionedFiles review cleanups
804
        :return: As much of a key (or prefix) as is derivable from the partition
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
805
            id.
806
        """
807
        raise NotImplementedError(self.unmap)
808
809
810
class ConstantMapper(KeyMapper):
811
    """A key mapper that maps to a constant result."""
812
813
    def __init__(self, result):
814
        """Create a ConstantMapper which will return result for all maps."""
815
        self._result = result
816
817
    def map(self, key):
818
        """See KeyMapper.map()."""
819
        return self._result
820
821
822
class URLEscapeMapper(KeyMapper):
823
    """Base class for use with transport backed storage.
824
825
    This provides a map and unmap wrapper that respectively url escape and
826
    unescape their outputs and inputs.
827
    """
828
829
    def map(self, key):
830
        """See KeyMapper.map()."""
831
        return urllib.quote(self._map(key))
832
833
    def unmap(self, partition_id):
834
        """See KeyMapper.unmap()."""
835
        return self._unmap(urllib.unquote(partition_id))
836
837
838
class PrefixMapper(URLEscapeMapper):
839
    """A key mapper that extracts the first component of a key.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
840
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
841
    This mapper is for use with a transport based backend.
842
    """
843
844
    def _map(self, key):
845
        """See KeyMapper.map()."""
846
        return key[0]
847
848
    def _unmap(self, partition_id):
849
        """See KeyMapper.unmap()."""
850
        return (partition_id,)
851
852
853
class HashPrefixMapper(URLEscapeMapper):
854
    """A key mapper that combines the first component of a key with a hash.
855
856
    This mapper is for use with a transport based backend.
857
    """
858
859
    def _map(self, key):
860
        """See KeyMapper.map()."""
861
        prefix = self._escape(key[0])
862
        return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
863
864
    def _escape(self, prefix):
865
        """No escaping needed here."""
866
        return prefix
867
868
    def _unmap(self, partition_id):
869
        """See KeyMapper.unmap()."""
870
        return (self._unescape(osutils.basename(partition_id)),)
871
872
    def _unescape(self, basename):
873
        """No unescaping needed for HashPrefixMapper."""
874
        return basename
875
876
877
class HashEscapedPrefixMapper(HashPrefixMapper):
878
    """Combines the escaped first component of a key with a hash.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
879
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
880
    This mapper is for use with a transport based backend.
881
    """
882
883
    _safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
884
885
    def _escape(self, prefix):
886
        """Turn a key element into a filesystem safe string.
887
888
        This is similar to a plain urllib.quote, except
889
        it uses specific safe characters, so that it doesn't
890
        have to translate a lot of valid file ids.
891
        """
892
        # @ does not get escaped. This is because it is a valid
893
        # filesystem character we use all the time, and it looks
894
        # a lot better than seeing %40 all the time.
895
        r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
896
             for c in prefix]
897
        return ''.join(r)
898
899
    def _unescape(self, basename):
900
        """Escaped names are easily unescaped by urlutils."""
901
        return urllib.unquote(basename)
902
903
904
def make_versioned_files_factory(versioned_file_factory, mapper):
905
    """Create a ThunkedVersionedFiles factory.
906
907
    This will create a callable which when called creates a
908
    ThunkedVersionedFiles on a transport, using mapper to access individual
909
    versioned files, and versioned_file_factory to create each individual file.
910
    """
911
    def factory(transport):
912
        return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
913
            lambda:True)
914
    return factory
915
916
917
class VersionedFiles(object):
918
    """Storage for many versioned files.
919
920
    This object allows a single keyspace for accessing the history graph and
921
    contents of named bytestrings.
922
923
    Currently no implementation allows the graph of different key prefixes to
924
    intersect, but the API does allow such implementations in the future.
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
925
926
    The keyspace is expressed via simple tuples. Any instance of VersionedFiles
927
    may have a different length key-size, but that size will be constant for
928
    all texts added to or retrieved from it. For instance, bzrlib uses
929
    instances with a key-size of 2 for storing user files in a repository, with
930
    the first element the fileid, and the second the version of that file.
931
932
    The use of tuples allows a single code base to support several different
933
    uses with only the mapping logic changing from instance to instance.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
934
    """
935
936
    def add_lines(self, key, parents, lines, parent_texts=None,
937
        left_matching_blocks=None, nostore_sha=None, random_id=False,
938
        check_content=True):
939
        """Add a text to the store.
940
4241.4.1 by Ian Clatworthy
add sha generation support to versionedfiles
941
        :param key: The key tuple of the text to add. If the last element is
942
            None, a CHK string will be generated during the addition.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
943
        :param parents: The parents key tuples of the text to add.
944
        :param lines: A list of lines. Each line must be a bytestring. And all
945
            of them except the last must be terminated with \n and contain no
946
            other \n's. The last line may either contain no \n's or a single
947
            terminating \n. If the lines list does meet this constraint the add
948
            routine may error or may succeed - but you will be unable to read
949
            the data back accurately. (Checking the lines have been split
950
            correctly is expensive and extremely unlikely to catch bugs so it
951
            is not done at runtime unless check_content is True.)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
952
        :param parent_texts: An optional dictionary containing the opaque
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
953
            representations of some or all of the parents of version_id to
954
            allow delta optimisations.  VERY IMPORTANT: the texts must be those
955
            returned by add_lines or data corruption can be caused.
956
        :param left_matching_blocks: a hint about which areas are common
957
            between the text and its left-hand-parent.  The format is
958
            the SequenceMatcher.get_matching_blocks format.
959
        :param nostore_sha: Raise ExistingContent and do not add the lines to
960
            the versioned file if the digest of the lines matches this.
961
        :param random_id: If True a random id has been selected rather than
962
            an id determined by some deterministic process such as a converter
963
            from a foreign VCS. When True the backend may choose not to check
964
            for uniqueness of the resulting key within the versioned file, so
965
            this should only be done when the result is expected to be unique
966
            anyway.
967
        :param check_content: If True, the lines supplied are verified to be
968
            bytestrings that are correctly formed lines.
969
        :return: The text sha1, the number of bytes in the text, and an opaque
970
                 representation of the inserted version which can be provided
971
                 back to future add_lines calls in the parent_texts dictionary.
972
        """
973
        raise NotImplementedError(self.add_lines)
974
4398.8.6 by John Arbash Meinel
Switch the api from VF.add_text to VF._add_text and trim some extra 'features'.
975
    def _add_text(self, key, parents, text, nostore_sha=None, random_id=False):
976
        """Add a text to the store.
977
978
        This is a private function for use by CommitBuilder.
979
980
        :param key: The key tuple of the text to add. If the last element is
981
            None, a CHK string will be generated during the addition.
982
        :param parents: The parents key tuples of the text to add.
983
        :param text: A string containing the text to be committed.
984
        :param nostore_sha: Raise ExistingContent and do not add the lines to
985
            the versioned file if the digest of the lines matches this.
986
        :param random_id: If True a random id has been selected rather than
987
            an id determined by some deterministic process such as a converter
988
            from a foreign VCS. When True the backend may choose not to check
989
            for uniqueness of the resulting key within the versioned file, so
990
            this should only be done when the result is expected to be unique
991
            anyway.
992
        :param check_content: If True, the lines supplied are verified to be
993
            bytestrings that are correctly formed lines.
994
        :return: The text sha1, the number of bytes in the text, and an opaque
995
                 representation of the inserted version which can be provided
996
                 back to future _add_text calls in the parent_texts dictionary.
997
        """
998
        # The default implementation just thunks over to .add_lines(),
999
        # inefficient, but it works.
4398.8.1 by John Arbash Meinel
Add a VersionedFile.add_text() api.
1000
        return self.add_lines(key, parents, osutils.split_lines(text),
1001
                              nostore_sha=nostore_sha,
1002
                              random_id=random_id,
4398.8.6 by John Arbash Meinel
Switch the api from VF.add_text to VF._add_text and trim some extra 'features'.
1003
                              check_content=True)
4398.8.1 by John Arbash Meinel
Add a VersionedFile.add_text() api.
1004
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1005
    def add_mpdiffs(self, records):
1006
        """Add mpdiffs to this VersionedFile.
1007
1008
        Records should be iterables of version, parents, expected_sha1,
1009
        mpdiff. mpdiff should be a MultiParent instance.
1010
        """
1011
        vf_parents = {}
1012
        mpvf = multiparent.MultiMemoryVersionedFile()
1013
        versions = []
1014
        for version, parent_ids, expected_sha1, mpdiff in records:
1015
            versions.append(version)
1016
            mpvf.add_diff(mpdiff, version, parent_ids)
1017
        needed_parents = set()
1018
        for version, parent_ids, expected_sha1, mpdiff in records:
1019
            needed_parents.update(p for p in parent_ids
1020
                                  if not mpvf.has_version(p))
1021
        # It seems likely that adding all the present parents as fulltexts can
1022
        # easily exhaust memory.
3890.2.9 by John Arbash Meinel
Start using osutils.chunks_as_lines rather than osutils.split_lines.
1023
        chunks_to_lines = osutils.chunks_to_lines
3350.8.11 by Robert Collins
Stacked add_mpdiffs.
1024
        for record in self.get_record_stream(needed_parents, 'unordered',
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1025
            True):
3350.8.11 by Robert Collins
Stacked add_mpdiffs.
1026
            if record.storage_kind == 'absent':
1027
                continue
3890.2.9 by John Arbash Meinel
Start using osutils.chunks_as_lines rather than osutils.split_lines.
1028
            mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1029
                record.key, [])
1030
        for (key, parent_keys, expected_sha1, mpdiff), lines in\
1031
            zip(records, mpvf.get_line_list(versions)):
1032
            if len(parent_keys) == 1:
1033
                left_matching_blocks = list(mpdiff.get_matching_blocks(0,
1034
                    mpvf.get_diff(parent_keys[0]).num_lines()))
1035
            else:
1036
                left_matching_blocks = None
1037
            version_sha1, _, version_text = self.add_lines(key,
1038
                parent_keys, lines, vf_parents,
1039
                left_matching_blocks=left_matching_blocks)
1040
            if version_sha1 != expected_sha1:
1041
                raise errors.VersionedFileInvalidChecksum(version)
1042
            vf_parents[key] = version_text
1043
1044
    def annotate(self, key):
1045
        """Return a list of (version-key, line) tuples for the text of key.
1046
1047
        :raise RevisionNotPresent: If the key is not present.
1048
        """
1049
        raise NotImplementedError(self.annotate)
1050
1051
    def check(self, progress_bar=None):
4332.3.26 by Robert Collins
Allow passing keys to check to VersionedFile.check().
1052
        """Check this object for integrity.
1053
        
1054
        :param progress_bar: A progress bar to output as the check progresses.
1055
        :param keys: Specific keys within the VersionedFiles to check. When
1056
            this parameter is not None, check() becomes a generator as per
1057
            get_record_stream. The difference to get_record_stream is that
1058
            more or deeper checks will be performed.
1059
        :return: None, or if keys was supplied a generator as per
1060
            get_record_stream.
1061
        """
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1062
        raise NotImplementedError(self.check)
1063
1064
    @staticmethod
1065
    def check_not_reserved_id(version_id):
1066
        revision.check_not_reserved_id(version_id)
1067
4744.2.5 by John Arbash Meinel
Change to a generic 'VersionedFiles.clear_cache()' api.
1068
    def clear_cache(self):
1069
        """Clear whatever caches this VersionedFile holds.
1070
1071
        This is generally called after an operation has been performed, when we
1072
        don't expect to be using this versioned file again soon.
1073
        """
1074
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1075
    def _check_lines_not_unicode(self, lines):
1076
        """Check that lines being added to a versioned file are not unicode."""
1077
        for line in lines:
1078
            if line.__class__ is not str:
1079
                raise errors.BzrBadParameterUnicode("lines")
1080
1081
    def _check_lines_are_lines(self, lines):
1082
        """Check that the lines really are full lines without inline EOL."""
1083
        for line in lines:
1084
            if '\n' in line[:-1]:
1085
                raise errors.BzrBadParameterContainsNewline("lines")
1086
4593.5.36 by John Arbash Meinel
a few more implementations of the interface.
1087
    def get_known_graph_ancestry(self, keys):
1088
        """Get a KnownGraph instance with the ancestry of keys."""
1089
        # most basic implementation is a loop around get_parent_map
1090
        pending = set(keys)
1091
        parent_map = {}
1092
        while pending:
1093
            this_parent_map = self.get_parent_map(pending)
1094
            parent_map.update(this_parent_map)
1095
            pending = set()
1096
            map(pending.update, this_parent_map.itervalues())
1097
            pending = pending.difference(parent_map)
1098
        kg = _mod_graph.KnownGraph(parent_map)
1099
        return kg
1100
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1101
    def get_parent_map(self, keys):
1102
        """Get a map of the parents of keys.
1103
1104
        :param keys: The keys to look up parents for.
1105
        :return: A mapping from keys to parents. Absent keys are absent from
1106
            the mapping.
1107
        """
1108
        raise NotImplementedError(self.get_parent_map)
1109
1110
    def get_record_stream(self, keys, ordering, include_delta_closure):
1111
        """Get a stream of records for keys.
1112
1113
        :param keys: The keys to include.
1114
        :param ordering: Either 'unordered' or 'topological'. A topologically
1115
            sorted stream has compression parents strictly before their
1116
            children.
1117
        :param include_delta_closure: If True then the closure across any
1118
            compression parents will be included (in the opaque data).
1119
        :return: An iterator of ContentFactory objects, each of which is only
1120
            valid until the iterator is advanced.
1121
        """
1122
        raise NotImplementedError(self.get_record_stream)
1123
1124
    def get_sha1s(self, keys):
1125
        """Get the sha1's of the texts for the given keys.
1126
1127
        :param keys: The names of the keys to lookup
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1128
        :return: a dict from key to sha1 digest. Keys of texts which are not
3350.8.14 by Robert Collins
Review feedback.
1129
            present in the store are not present in the returned
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1130
            dictionary.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1131
        """
1132
        raise NotImplementedError(self.get_sha1s)
1133
3830.3.12 by Martin Pool
Review cleanups: unify has_key impls, add missing_keys(), clean up exception blocks
1134
    has_key = index._has_key_from_parent_map
1135
4009.3.3 by Andrew Bennetts
Add docstrings.
1136
    def get_missing_compression_parent_keys(self):
1137
        """Return an iterable of keys of missing compression parents.
1138
1139
        Check this after calling insert_record_stream to find out if there are
1140
        any missing compression parents.  If there are, the records that
4009.3.12 by Robert Collins
Polish on inserting record streams with missing compression parents.
1141
        depend on them are not able to be inserted safely. The precise
1142
        behaviour depends on the concrete VersionedFiles class in use.
1143
1144
        Classes that do not support this will raise NotImplementedError.
4009.3.3 by Andrew Bennetts
Add docstrings.
1145
        """
1146
        raise NotImplementedError(self.get_missing_compression_parent_keys)
1147
5195.3.26 by Parth Malwankar
reverted changes done to insert_record_stream API
1148
    def insert_record_stream(self, stream):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1149
        """Insert a record stream into this container.
1150
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1151
        :param stream: A stream of records to insert.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1152
        :return: None
1153
        :seealso VersionedFile.get_record_stream:
1154
        """
1155
        raise NotImplementedError
1156
1157
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1158
        """Iterate over the lines in the versioned files from keys.
1159
1160
        This may return lines from other keys. Each item the returned
1161
        iterator yields is a tuple of a line and a text version that that line
1162
        is present in (not introduced in).
1163
1164
        Ordering of results is in whatever order is most suitable for the
1165
        underlying storage format.
1166
1167
        If a progress bar is supplied, it may be used to indicate progress.
1168
        The caller is responsible for cleaning up progress bars (because this
1169
        is an iterator).
1170
1171
        NOTES:
1172
         * Lines are normalised by the underlying store: they will all have \n
1173
           terminators.
1174
         * Lines are returned in arbitrary order.
1175
1176
        :return: An iterator over (line, key).
1177
        """
1178
        raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
1179
1180
    def keys(self):
1181
        """Return a iterable of the keys for all the contained texts."""
1182
        raise NotImplementedError(self.keys)
1183
1184
    def make_mpdiffs(self, keys):
1185
        """Create multiparent diffs for specified keys."""
5374.2.3 by John Arbash Meinel
Replace the generic VersionedFiles.make_mpdiffs interface.
1186
        import pdb; pdb.set_trace()
1187
        generator = _MPDiffGenerator(self, keys)
1188
        return generator.compute_diffs()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1189
5363.1.1 by Jelmer Vernooij
Provide VersionedFiles.get_annotator.
1190
    def get_annotator(self):
1191
        return annotate.Annotator(self)
1192
3830.3.12 by Martin Pool
Review cleanups: unify has_key impls, add missing_keys(), clean up exception blocks
1193
    missing_keys = index._missing_keys_from_parent_map
1194
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1195
    def _extract_blocks(self, version_id, source, target):
1196
        return None
1197
1198
1199
class ThunkedVersionedFiles(VersionedFiles):
1200
    """Storage for many versioned files thunked onto a 'VersionedFile' class.
1201
1202
    This object allows a single keyspace for accessing the history graph and
1203
    contents of named bytestrings.
1204
1205
    Currently no implementation allows the graph of different key prefixes to
1206
    intersect, but the API does allow such implementations in the future.
1207
    """
1208
1209
    def __init__(self, transport, file_factory, mapper, is_locked):
1210
        """Create a ThunkedVersionedFiles."""
1211
        self._transport = transport
1212
        self._file_factory = file_factory
1213
        self._mapper = mapper
1214
        self._is_locked = is_locked
1215
1216
    def add_lines(self, key, parents, lines, parent_texts=None,
1217
        left_matching_blocks=None, nostore_sha=None, random_id=False,
1218
        check_content=True):
1219
        """See VersionedFiles.add_lines()."""
1220
        path = self._mapper.map(key)
1221
        version_id = key[-1]
1222
        parents = [parent[-1] for parent in parents]
1223
        vf = self._get_vf(path)
1224
        try:
1225
            try:
1226
                return vf.add_lines_with_ghosts(version_id, parents, lines,
1227
                    parent_texts=parent_texts,
1228
                    left_matching_blocks=left_matching_blocks,
1229
                    nostore_sha=nostore_sha, random_id=random_id,
1230
                    check_content=check_content)
1231
            except NotImplementedError:
1232
                return vf.add_lines(version_id, parents, lines,
1233
                    parent_texts=parent_texts,
1234
                    left_matching_blocks=left_matching_blocks,
1235
                    nostore_sha=nostore_sha, random_id=random_id,
1236
                    check_content=check_content)
1237
        except errors.NoSuchFile:
1238
            # parent directory may be missing, try again.
1239
            self._transport.mkdir(osutils.dirname(path))
1240
            try:
1241
                return vf.add_lines_with_ghosts(version_id, parents, lines,
1242
                    parent_texts=parent_texts,
1243
                    left_matching_blocks=left_matching_blocks,
1244
                    nostore_sha=nostore_sha, random_id=random_id,
1245
                    check_content=check_content)
1246
            except NotImplementedError:
1247
                return vf.add_lines(version_id, parents, lines,
1248
                    parent_texts=parent_texts,
1249
                    left_matching_blocks=left_matching_blocks,
1250
                    nostore_sha=nostore_sha, random_id=random_id,
1251
                    check_content=check_content)
1252
1253
    def annotate(self, key):
1254
        """Return a list of (version-key, line) tuples for the text of key.
1255
1256
        :raise RevisionNotPresent: If the key is not present.
1257
        """
1258
        prefix = key[:-1]
1259
        path = self._mapper.map(prefix)
1260
        vf = self._get_vf(path)
1261
        origins = vf.annotate(key[-1])
1262
        result = []
1263
        for origin, line in origins:
1264
            result.append((prefix + (origin,), line))
1265
        return result
1266
4332.3.26 by Robert Collins
Allow passing keys to check to VersionedFile.check().
1267
    def check(self, progress_bar=None, keys=None):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1268
        """See VersionedFiles.check()."""
4332.3.26 by Robert Collins
Allow passing keys to check to VersionedFile.check().
1269
        # XXX: This is over-enthusiastic but as we only thunk for Weaves today
1270
        # this is tolerable. Ideally we'd pass keys down to check() and 
1271
        # have the older VersiondFile interface updated too.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1272
        for prefix, vf in self._iter_all_components():
1273
            vf.check()
4332.3.26 by Robert Collins
Allow passing keys to check to VersionedFile.check().
1274
        if keys is not None:
1275
            return self.get_record_stream(keys, 'unordered', True)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1276
1277
    def get_parent_map(self, keys):
1278
        """Get a map of the parents of keys.
1279
1280
        :param keys: The keys to look up parents for.
1281
        :return: A mapping from keys to parents. Absent keys are absent from
1282
            the mapping.
1283
        """
1284
        prefixes = self._partition_keys(keys)
1285
        result = {}
1286
        for prefix, suffixes in prefixes.items():
1287
            path = self._mapper.map(prefix)
1288
            vf = self._get_vf(path)
1289
            parent_map = vf.get_parent_map(suffixes)
1290
            for key, parents in parent_map.items():
1291
                result[prefix + (key,)] = tuple(
1292
                    prefix + (parent,) for parent in parents)
1293
        return result
1294
1295
    def _get_vf(self, path):
1296
        if not self._is_locked():
1297
            raise errors.ObjectNotLocked(self)
1298
        return self._file_factory(path, self._transport, create=True,
1299
            get_scope=lambda:None)
1300
1301
    def _partition_keys(self, keys):
1302
        """Turn keys into a dict of prefix:suffix_list."""
1303
        result = {}
1304
        for key in keys:
1305
            prefix_keys = result.setdefault(key[:-1], [])
1306
            prefix_keys.append(key[-1])
1307
        return result
1308
1309
    def _get_all_prefixes(self):
1310
        # Identify all key prefixes.
1311
        # XXX: A bit hacky, needs polish.
1312
        if type(self._mapper) == ConstantMapper:
1313
            paths = [self._mapper.map(())]
1314
            prefixes = [()]
1315
        else:
1316
            relpaths = set()
1317
            for quoted_relpath in self._transport.iter_files_recursive():
1318
                path, ext = os.path.splitext(quoted_relpath)
1319
                relpaths.add(path)
1320
            paths = list(relpaths)
1321
            prefixes = [self._mapper.unmap(path) for path in paths]
1322
        return zip(paths, prefixes)
1323
1324
    def get_record_stream(self, keys, ordering, include_delta_closure):
1325
        """See VersionedFiles.get_record_stream()."""
1326
        # Ordering will be taken care of by each partitioned store; group keys
1327
        # by partition.
1328
        keys = sorted(keys)
1329
        for prefix, suffixes, vf in self._iter_keys_vf(keys):
1330
            suffixes = [(suffix,) for suffix in suffixes]
1331
            for record in vf.get_record_stream(suffixes, ordering,
1332
                include_delta_closure):
1333
                if record.parents is not None:
1334
                    record.parents = tuple(
1335
                        prefix + parent for parent in record.parents)
1336
                record.key = prefix + record.key
1337
                yield record
1338
1339
    def _iter_keys_vf(self, keys):
1340
        prefixes = self._partition_keys(keys)
1341
        sha1s = {}
1342
        for prefix, suffixes in prefixes.items():
1343
            path = self._mapper.map(prefix)
1344
            vf = self._get_vf(path)
1345
            yield prefix, suffixes, vf
1346
1347
    def get_sha1s(self, keys):
1348
        """See VersionedFiles.get_sha1s()."""
1349
        sha1s = {}
1350
        for prefix,suffixes, vf in self._iter_keys_vf(keys):
1351
            vf_sha1s = vf.get_sha1s(suffixes)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1352
            for suffix, sha1 in vf_sha1s.iteritems():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1353
                sha1s[prefix + (suffix,)] = sha1
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1354
        return sha1s
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1355
5195.3.26 by Parth Malwankar
reverted changes done to insert_record_stream API
1356
    def insert_record_stream(self, stream):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1357
        """Insert a record stream into this container.
1358
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1359
        :param stream: A stream of records to insert.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1360
        :return: None
1361
        :seealso VersionedFile.get_record_stream:
1362
        """
1363
        for record in stream:
1364
            prefix = record.key[:-1]
1365
            key = record.key[-1:]
1366
            if record.parents is not None:
1367
                parents = [parent[-1:] for parent in record.parents]
1368
            else:
1369
                parents = None
1370
            thunk_record = AdapterFactory(key, parents, record)
1371
            path = self._mapper.map(prefix)
1372
            # Note that this parses the file many times; we can do better but
1373
            # as this only impacts weaves in terms of performance, it is
1374
            # tolerable.
1375
            vf = self._get_vf(path)
1376
            vf.insert_record_stream([thunk_record])
1377
1378
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1379
        """Iterate over the lines in the versioned files from keys.
1380
1381
        This may return lines from other keys. Each item the returned
1382
        iterator yields is a tuple of a line and a text version that that line
1383
        is present in (not introduced in).
1384
1385
        Ordering of results is in whatever order is most suitable for the
1386
        underlying storage format.
1387
1388
        If a progress bar is supplied, it may be used to indicate progress.
1389
        The caller is responsible for cleaning up progress bars (because this
1390
        is an iterator).
1391
1392
        NOTES:
1393
         * Lines are normalised by the underlying store: they will all have \n
1394
           terminators.
1395
         * Lines are returned in arbitrary order.
1396
1397
        :return: An iterator over (line, key).
1398
        """
1399
        for prefix, suffixes, vf in self._iter_keys_vf(keys):
1400
            for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1401
                yield line, prefix + (version,)
1402
1403
    def _iter_all_components(self):
1404
        for path, prefix in self._get_all_prefixes():
1405
            yield prefix, self._get_vf(path)
1406
1407
    def keys(self):
1408
        """See VersionedFiles.keys()."""
1409
        result = set()
1410
        for prefix, vf in self._iter_all_components():
1411
            for suffix in vf.versions():
1412
                result.add(prefix + (suffix,))
1413
        return result
1414
1415
1416
class _PlanMergeVersionedFile(VersionedFiles):
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1417
    """A VersionedFile for uncommitted and committed texts.
1418
1419
    It is intended to allow merges to be planned with working tree texts.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1420
    It implements only the small part of the VersionedFiles interface used by
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1421
    PlanMerge.  It falls back to multiple versionedfiles for data not stored in
1422
    _PlanMergeVersionedFile itself.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1423
1424
    :ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1425
        queried for missing texts.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1426
    """
1427
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1428
    def __init__(self, file_id):
1429
        """Create a _PlanMergeVersionedFile.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1430
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1431
        :param file_id: Used with _PlanMerge code which is not yet fully
1432
            tuple-keyspace aware.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1433
        """
1434
        self._file_id = file_id
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1435
        # fallback locations
1436
        self.fallback_versionedfiles = []
1437
        # Parents for locally held keys.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1438
        self._parents = {}
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1439
        # line data for locally held keys.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1440
        self._lines = {}
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1441
        # key lookup providers
1442
        self._providers = [DictParentsProvider(self._parents)]
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1443
3062.2.3 by Aaron Bentley
Sync up with bzr.dev API changes
1444
    def plan_merge(self, ver_a, ver_b, base=None):
3062.1.13 by Aaron Bentley
Make _PlanMerge an implementation detail of _PlanMergeVersionedFile
1445
        """See VersionedFile.plan_merge"""
3144.3.7 by Aaron Bentley
Update from review
1446
        from bzrlib.merge import _PlanMerge
3062.2.3 by Aaron Bentley
Sync up with bzr.dev API changes
1447
        if base is None:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1448
            return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1449
        old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1450
        new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
3062.2.3 by Aaron Bentley
Sync up with bzr.dev API changes
1451
        return _PlanMerge._subtract_plans(old_plan, new_plan)
1452
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1453
    def plan_lca_merge(self, ver_a, ver_b, base=None):
3144.3.7 by Aaron Bentley
Update from review
1454
        from bzrlib.merge import _PlanLCAMerge
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1455
        graph = Graph(self)
1456
        new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1457
        if base is None:
1458
            return new_plan
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1459
        old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1460
        return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
3062.1.13 by Aaron Bentley
Make _PlanMerge an implementation detail of _PlanMergeVersionedFile
1461
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1462
    def add_lines(self, key, parents, lines):
1463
        """See VersionedFiles.add_lines
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1464
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1465
        Lines are added locally, not to fallback versionedfiles.  Also, ghosts
1466
        are permitted.  Only reserved ids are permitted.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1467
        """
3350.6.8 by Martin Pool
Change stray pdb calls to exceptions
1468
        if type(key) is not tuple:
1469
            raise TypeError(key)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1470
        if not revision.is_reserved_id(key[-1]):
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1471
            raise ValueError('Only reserved ids may be used')
1472
        if parents is None:
1473
            raise ValueError('Parents may not be None')
1474
        if lines is None:
1475
            raise ValueError('Lines may not be None')
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1476
        self._parents[key] = tuple(parents)
1477
        self._lines[key] = lines
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1478
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1479
    def get_record_stream(self, keys, ordering, include_delta_closure):
1480
        pending = set(keys)
1481
        for key in keys:
1482
            if key in self._lines:
1483
                lines = self._lines[key]
1484
                parents = self._parents[key]
1485
                pending.remove(key)
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
1486
                yield ChunkedContentFactory(key, parents, None, lines)
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1487
        for versionedfile in self.fallback_versionedfiles:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1488
            for record in versionedfile.get_record_stream(
1489
                pending, 'unordered', True):
1490
                if record.storage_kind == 'absent':
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1491
                    continue
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1492
                else:
1493
                    pending.remove(record.key)
1494
                    yield record
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
1495
            if not pending:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1496
                return
1497
        # report absent entries
1498
        for key in pending:
1499
            yield AbsentContentFactory(key)
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1500
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1501
    def get_parent_map(self, keys):
1502
        """See VersionedFiles.get_parent_map"""
1503
        # We create a new provider because a fallback may have been added.
1504
        # If we make fallbacks private we can update a stack list and avoid
1505
        # object creation thrashing.
3350.6.6 by Robert Collins
Fix test_plan_file_merge
1506
        keys = set(keys)
1507
        result = {}
1508
        if revision.NULL_REVISION in keys:
1509
            keys.remove(revision.NULL_REVISION)
1510
            result[revision.NULL_REVISION] = ()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1511
        self._providers = self._providers[:1] + self.fallback_versionedfiles
3350.6.6 by Robert Collins
Fix test_plan_file_merge
1512
        result.update(
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
1513
            StackedParentsProvider(self._providers).get_parent_map(keys))
3350.6.5 by Robert Collins
Update to bzr.dev.
1514
        for key, parents in result.iteritems():
1515
            if parents == ():
1516
                result[key] = (revision.NULL_REVISION,)
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
1517
        return result
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1518
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1519
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1520
class PlanWeaveMerge(TextMerge):
1551.6.13 by Aaron Bentley
Cleanup
1521
    """Weave merge that takes a plan as its input.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1522
1551.6.14 by Aaron Bentley
Tweaks from merge review
1523
    This exists so that VersionedFile.plan_merge is implementable.
1524
    Most callers will want to use WeaveMerge instead.
1551.6.13 by Aaron Bentley
Cleanup
1525
    """
1526
1551.6.14 by Aaron Bentley
Tweaks from merge review
1527
    def __init__(self, plan, a_marker=TextMerge.A_MARKER,
1528
                 b_marker=TextMerge.B_MARKER):
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1529
        TextMerge.__init__(self, a_marker, b_marker)
4634.101.7 by John Arbash Meinel
Start working on the ability to drop a .BASE file for --weave and --lca merge.
1530
        self.plan = list(plan)
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1531
1551.6.7 by Aaron Bentley
Implemented two-way merge, refactored weave merge
1532
    def _merge_struct(self):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1533
        lines_a = []
1534
        lines_b = []
1535
        ch_a = ch_b = False
1664.2.8 by Aaron Bentley
Fix WeaveMerge when plan doesn't end with unchanged lines
1536
1537
        def outstanding_struct():
1538
            if not lines_a and not lines_b:
1539
                return
1540
            elif ch_a and not ch_b:
1541
                # one-sided change:
1542
                yield(lines_a,)
1543
            elif ch_b and not ch_a:
1544
                yield (lines_b,)
1545
            elif lines_a == lines_b:
1546
                yield(lines_a,)
1547
            else:
1548
                yield (lines_a, lines_b)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1549
1616.1.18 by Martin Pool
(weave-merge) don't treat killed-both lines as points of agreement;
1550
        # We previously considered either 'unchanged' or 'killed-both' lines
1551
        # to be possible places to resynchronize.  However, assuming agreement
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
1552
        # on killed-both lines may be too aggressive. -- mbp 20060324
1551.6.7 by Aaron Bentley
Implemented two-way merge, refactored weave merge
1553
        for state, line in self.plan:
1616.1.18 by Martin Pool
(weave-merge) don't treat killed-both lines as points of agreement;
1554
            if state == 'unchanged':
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1555
                # resync and flush queued conflicts changes if any
1664.2.8 by Aaron Bentley
Fix WeaveMerge when plan doesn't end with unchanged lines
1556
                for struct in outstanding_struct():
1557
                    yield struct
1551.6.11 by Aaron Bentley
Switched TextMerge_lines to work on a list
1558
                lines_a = []
1559
                lines_b = []
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1560
                ch_a = ch_b = False
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1561
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1562
            if state == 'unchanged':
1563
                if line:
1551.6.5 by Aaron Bentley
Got weave merge producing structural output
1564
                    yield ([line],)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1565
            elif state == 'killed-a':
1566
                ch_a = True
1567
                lines_b.append(line)
1568
            elif state == 'killed-b':
1569
                ch_b = True
1570
                lines_a.append(line)
1571
            elif state == 'new-a':
1572
                ch_a = True
1573
                lines_a.append(line)
1574
            elif state == 'new-b':
1575
                ch_b = True
1576
                lines_b.append(line)
3144.3.2 by Aaron Bentley
Get conflict handling working
1577
            elif state == 'conflicted-a':
1578
                ch_b = ch_a = True
1579
                lines_a.append(line)
1580
            elif state == 'conflicted-b':
1581
                ch_b = ch_a = True
1582
                lines_b.append(line)
4312.1.1 by John Arbash Meinel
Add a per-implementation test that deleting lines conflicts with modifying lines.
1583
            elif state == 'killed-both':
1584
                # This counts as a change, even though there is no associated
1585
                # line
1586
                ch_b = ch_a = True
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1587
            else:
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1588
                if state not in ('irrelevant', 'ghost-a', 'ghost-b',
4312.1.1 by John Arbash Meinel
Add a per-implementation test that deleting lines conflicts with modifying lines.
1589
                        'killed-base'):
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1590
                    raise AssertionError(state)
1664.2.8 by Aaron Bentley
Fix WeaveMerge when plan doesn't end with unchanged lines
1591
        for struct in outstanding_struct():
1592
            yield struct
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
1593
4634.101.7 by John Arbash Meinel
Start working on the ability to drop a .BASE file for --weave and --lca merge.
1594
    def base_from_plan(self):
1595
        """Construct a BASE file from the plan text."""
1596
        base_lines = []
1597
        for state, line in self.plan:
4634.101.9 by John Arbash Meinel
Reverse the .BASE values for --lca and conflicted lines.
1598
            if state in ('killed-a', 'killed-b', 'killed-both', 'unchanged'):
4634.101.7 by John Arbash Meinel
Start working on the ability to drop a .BASE file for --weave and --lca merge.
1599
                # If unchanged, then this line is straight from base. If a or b
1600
                # or both killed the line, then it *used* to be in base.
1601
                base_lines.append(line)
1602
            else:
1603
                if state not in ('killed-base', 'irrelevant',
1604
                                 'ghost-a', 'ghost-b',
4634.101.9 by John Arbash Meinel
Reverse the .BASE values for --lca and conflicted lines.
1605
                                 'new-a', 'new-b',
1606
                                 'conflicted-a', 'conflicted-b'):
4634.101.7 by John Arbash Meinel
Start working on the ability to drop a .BASE file for --weave and --lca merge.
1607
                    # killed-base, irrelevant means it doesn't apply
1608
                    # ghost-a/ghost-b are harder to say for sure, but they
1609
                    # aren't in the 'inc_c' which means they aren't in the
4634.101.13 by John Arbash Meinel
Clean up a code comment.
1610
                    # shared base of a & b. So we don't include them.  And
1611
                    # obviously if the line is newly inserted, it isn't in base
1612
1613
                    # If 'conflicted-a' or b, then it is new vs one base, but
1614
                    # old versus another base. However, if we make it present
1615
                    # in the base, it will be deleted from the target, and it
4634.101.9 by John Arbash Meinel
Reverse the .BASE values for --lca and conflicted lines.
1616
                    # seems better to get a line doubled in the merge result,
1617
                    # rather than have it deleted entirely.
4634.101.13 by John Arbash Meinel
Clean up a code comment.
1618
                    # Example, each node is the 'text' at that point:
1619
                    #           MN
1620
                    #          /   \
1621
                    #        MaN   MbN
1622
                    #         |  X  |
1623
                    #        MabN MbaN
1624
                    #          \   /
1625
                    #           ???
1626
                    # There was a criss-cross conflict merge. Both sides
1627
                    # include the other, but put themselves first.
1628
                    # Weave marks this as a 'clean' merge, picking OTHER over
1629
                    # THIS. (Though the details depend on order inserted into
1630
                    # weave, etc.)
1631
                    # LCA generates a plan:
1632
                    # [('unchanged', M),
1633
                    #  ('conflicted-b', b),
1634
                    #  ('unchanged', a),
1635
                    #  ('conflicted-a', b),
1636
                    #  ('unchanged', N)]
1637
                    # If you mark 'conflicted-*' as part of BASE, then a 3-way
1638
                    # merge tool will cleanly generate "MaN" (as BASE vs THIS
1639
                    # removes one 'b', and BASE vs OTHER removes the other)
1640
                    # If you include neither, 3-way creates a clean "MbabN" as
1641
                    # THIS adds one 'b', and OTHER does too.
1642
                    # It seems that having the line 2 times is better than
1643
                    # having it omitted. (Easier to manually delete than notice
1644
                    # it needs to be added.)
4634.101.7 by John Arbash Meinel
Start working on the ability to drop a .BASE file for --weave and --lca merge.
1645
                    raise AssertionError('Unknown state: %s' % (state,))
1646
        return base_lines
1647
1664.2.14 by Aaron Bentley
spacing fix
1648
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1649
class WeaveMerge(PlanWeaveMerge):
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
1650
    """Weave merge that takes a VersionedFile and two versions as its input."""
1551.6.13 by Aaron Bentley
Cleanup
1651
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1652
    def __init__(self, versionedfile, ver_a, ver_b,
1551.6.14 by Aaron Bentley
Tweaks from merge review
1653
        a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1551.6.15 by Aaron Bentley
Moved plan_merge into Weave
1654
        plan = versionedfile.plan_merge(ver_a, ver_b)
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1655
        PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1656
1657
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1658
class VirtualVersionedFiles(VersionedFiles):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1659
    """Dummy implementation for VersionedFiles that uses other functions for
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1660
    obtaining fulltexts and parent maps.
1661
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1662
    This is always on the bottom of the stack and uses string keys
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1663
    (rather than tuples) internally.
1664
    """
1665
1666
    def __init__(self, get_parent_map, get_lines):
1667
        """Create a VirtualVersionedFiles.
1668
1669
        :param get_parent_map: Same signature as Repository.get_parent_map.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1670
        :param get_lines: Should return lines for specified key or None if
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1671
                          not available.
1672
        """
1673
        super(VirtualVersionedFiles, self).__init__()
1674
        self._get_parent_map = get_parent_map
1675
        self._get_lines = get_lines
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1676
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1677
    def check(self, progressbar=None):
1678
        """See VersionedFiles.check.
1679
1680
        :note: Always returns True for VirtualVersionedFiles.
1681
        """
1682
        return True
1683
1684
    def add_mpdiffs(self, records):
1685
        """See VersionedFiles.mpdiffs.
1686
1687
        :note: Not implemented for VirtualVersionedFiles.
1688
        """
1689
        raise NotImplementedError(self.add_mpdiffs)
1690
1691
    def get_parent_map(self, keys):
1692
        """See VersionedFiles.get_parent_map."""
3518.1.2 by Jelmer Vernooij
Fix some stylistic issues pointed out by Ian.
1693
        return dict([((k,), tuple([(p,) for p in v]))
1694
            for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1695
1696
    def get_sha1s(self, keys):
1697
        """See VersionedFiles.get_sha1s."""
1698
        ret = {}
1699
        for (k,) in keys:
1700
            lines = self._get_lines(k)
1701
            if lines is not None:
3518.1.2 by Jelmer Vernooij
Fix some stylistic issues pointed out by Ian.
1702
                if not isinstance(lines, list):
1703
                    raise AssertionError
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1704
                ret[(k,)] = osutils.sha_strings(lines)
1705
        return ret
1706
1707
    def get_record_stream(self, keys, ordering, include_delta_closure):
1708
        """See VersionedFiles.get_record_stream."""
1709
        for (k,) in list(keys):
1710
            lines = self._get_lines(k)
1711
            if lines is not None:
3518.1.2 by Jelmer Vernooij
Fix some stylistic issues pointed out by Ian.
1712
                if not isinstance(lines, list):
1713
                    raise AssertionError
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
1714
                yield ChunkedContentFactory((k,), None,
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1715
                        sha1=osutils.sha_strings(lines),
3890.2.1 by John Arbash Meinel
Start working on a ChunkedContentFactory.
1716
                        chunks=lines)
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1717
            else:
1718
                yield AbsentContentFactory((k,))
1719
3949.4.1 by Jelmer Vernooij
Implement VirtualVersionedFiles.iter_lines_added_or_present_in_keys.
1720
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1721
        """See VersionedFile.iter_lines_added_or_present_in_versions()."""
1722
        for i, (key,) in enumerate(keys):
1723
            if pb is not None:
4110.2.10 by Martin Pool
Tweak iter_lines progress messages
1724
                pb.update("Finding changed lines", i, len(keys))
3949.4.1 by Jelmer Vernooij
Implement VirtualVersionedFiles.iter_lines_added_or_present_in_keys.
1725
            for l in self._get_lines(key):
1726
                yield (l, key)
4005.3.2 by Robert Collins
First passing NetworkRecordStream test - a fulltext from any record type which isn't a chunked or fulltext can be serialised and deserialised successfully.
1727
1728
1729
def network_bytes_to_kind_and_offset(network_bytes):
1730
    """Strip of a record kind from the front of network_bytes.
1731
1732
    :param network_bytes: The bytes of a record.
1733
    :return: A tuple (storage_kind, offset_of_remaining_bytes)
1734
    """
1735
    line_end = network_bytes.find('\n')
1736
    storage_kind = network_bytes[:line_end]
1737
    return storage_kind, line_end + 1
1738
1739
1740
class NetworkRecordStream(object):
1741
    """A record_stream which reconstitures a serialised stream."""
1742
1743
    def __init__(self, bytes_iterator):
1744
        """Create a NetworkRecordStream.
1745
1746
        :param bytes_iterator: An iterator of bytes. Each item in this
1747
            iterator should have been obtained from a record_streams'
1748
            record.get_bytes_as(record.storage_kind) call.
1749
        """
1750
        self._bytes_iterator = bytes_iterator
4476.3.4 by Andrew Bennetts
Network serialisation, and most tests passing with InterDifferingSerializer commented out.
1751
        self._kind_factory = {
1752
            'fulltext': fulltext_network_to_record,
1753
            'groupcompress-block': groupcompress.network_block_to_records,
1754
            'knit-ft-gz': knit.knit_network_to_record,
1755
            'knit-delta-gz': knit.knit_network_to_record,
1756
            'knit-annotated-ft-gz': knit.knit_network_to_record,
1757
            'knit-annotated-delta-gz': knit.knit_network_to_record,
1758
            'knit-delta-closure': knit.knit_delta_closure_to_records,
4005.3.2 by Robert Collins
First passing NetworkRecordStream test - a fulltext from any record type which isn't a chunked or fulltext can be serialised and deserialised successfully.
1759
            }
1760
1761
    def read(self):
1762
        """Read the stream.
1763
1764
        :return: An iterator as per VersionedFiles.get_record_stream().
1765
        """
1766
        for bytes in self._bytes_iterator:
1767
            storage_kind, line_end = network_bytes_to_kind_and_offset(bytes)
4005.3.6 by Robert Collins
Support delta_closure=True with NetworkRecordStream to transmit deltas over the wire when full text extraction is required on the far end.
1768
            for record in self._kind_factory[storage_kind](
1769
                storage_kind, bytes, line_end):
1770
                yield record
4022.1.6 by Robert Collins
Cherrypick and polish the RemoteSink for streaming push.
1771
1772
1773
def fulltext_network_to_record(kind, bytes, line_end):
1774
    """Convert a network fulltext record to record."""
1775
    meta_len, = struct.unpack('!L', bytes[line_end:line_end+4])
4060.1.4 by Robert Collins
Streaming fetch from remote servers.
1776
    record_meta = bytes[line_end+4:line_end+4+meta_len]
4022.1.6 by Robert Collins
Cherrypick and polish the RemoteSink for streaming push.
1777
    key, parents = bencode.bdecode_as_tuple(record_meta)
1778
    if parents == 'nil':
1779
        parents = None
4060.1.4 by Robert Collins
Streaming fetch from remote servers.
1780
    fulltext = bytes[line_end+4+meta_len:]
1781
    return [FulltextContentFactory(key, parents, None, fulltext)]
4022.1.6 by Robert Collins
Cherrypick and polish the RemoteSink for streaming push.
1782
1783
1784
def _length_prefix(bytes):
1785
    return struct.pack('!L', len(bytes))
1786
1787
4060.1.4 by Robert Collins
Streaming fetch from remote servers.
1788
def record_to_fulltext_bytes(record):
4022.1.6 by Robert Collins
Cherrypick and polish the RemoteSink for streaming push.
1789
    if record.parents is None:
1790
        parents = 'nil'
1791
    else:
1792
        parents = record.parents
1793
    record_meta = bencode.bencode((record.key, parents))
1794
    record_content = record.get_bytes_as('fulltext')
1795
    return "fulltext\n%s%s%s" % (
1796
        _length_prefix(record_meta), record_meta, record_content)
4111.1.1 by Robert Collins
Add a groupcompress sort order.
1797
1798
1799
def sort_groupcompress(parent_map):
1800
    """Sort and group the keys in parent_map into groupcompress order.
1801
1802
    groupcompress is defined (currently) as reverse-topological order, grouped
1803
    by the key prefix.
1804
1805
    :return: A sorted-list of keys
1806
    """
1807
    # gc-optimal ordering is approximately reverse topological,
1808
    # properly grouped by file-id.
1809
    per_prefix_map = {}
1810
    for item in parent_map.iteritems():
1811
        key = item[0]
1812
        if isinstance(key, str) or len(key) == 1:
1813
            prefix = ''
1814
        else:
1815
            prefix = key[0]
1816
        try:
1817
            per_prefix_map[prefix].append(item)
1818
        except KeyError:
1819
            per_prefix_map[prefix] = [item]
1820
1821
    present_keys = []
1822
    for prefix in sorted(per_prefix_map):
1823
        present_keys.extend(reversed(tsort.topo_sort(per_prefix_map[prefix])))
1824
    return present_keys