/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
2052.3.2 by John Arbash Meinel
Change Copyright .. by Canonical to Copyright ... Canonical
1
# Copyright (C) 2005, 2006 Canonical Ltd
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
2
#
3
# Authors:
4
#   Johan Rydberg <jrydberg@gnu.org>
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
10
#
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
15
#
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
19
20
"""Versioned text file storage api."""
21
3350.8.2 by Robert Collins
stacked get_parent_map.
22
from copy import copy
3350.6.1 by Robert Collins
* New ``versionedfile.KeyMapper`` interface to abstract out the access to
23
from cStringIO import StringIO
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
24
import os
3350.6.1 by Robert Collins
* New ``versionedfile.KeyMapper`` interface to abstract out the access to
25
from zlib import adler32
26
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
27
from bzrlib.lazy_import import lazy_import
28
lazy_import(globals(), """
3224.5.20 by Andrew Bennetts
Remove or lazyify a couple more imports.
29
import urllib
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
30
31
from bzrlib import (
32
    errors,
2249.5.12 by John Arbash Meinel
Change the APIs for VersionedFile, Store, and some of Repository into utf-8
33
    osutils,
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
34
    multiparent,
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
35
    tsort,
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
36
    revision,
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
37
    ui,
38
    )
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
39
from bzrlib.graph import DictParentsProvider, Graph, _StackedParentsProvider
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
40
from bzrlib.transport.memory import MemoryTransport
41
""")
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
42
from bzrlib.inter import InterObject
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
43
from bzrlib.registry import Registry
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
44
from bzrlib.symbol_versioning import *
1551.6.7 by Aaron Bentley
Implemented two-way merge, refactored weave merge
45
from bzrlib.textmerge import TextMerge
1563.2.11 by Robert Collins
Consolidate reweave and join as we have no separate usage, make reweave tests apply to all versionedfile implementations and deprecate the old reweave apis.
46
47
3350.3.7 by Robert Collins
Create a registry of versioned file record adapters.
48
adapter_registry = Registry()
49
adapter_registry.register_lazy(('knit-delta-gz', 'fulltext'), 'bzrlib.knit',
50
    'DeltaPlainToFullText')
51
adapter_registry.register_lazy(('knit-ft-gz', 'fulltext'), 'bzrlib.knit',
52
    'FTPlainToFullText')
53
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'knit-delta-gz'),
54
    'bzrlib.knit', 'DeltaAnnotatedToUnannotated')
55
adapter_registry.register_lazy(('knit-annotated-delta-gz', 'fulltext'),
56
    'bzrlib.knit', 'DeltaAnnotatedToFullText')
57
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'knit-ft-gz'),
58
    'bzrlib.knit', 'FTAnnotatedToUnannotated')
59
adapter_registry.register_lazy(('knit-annotated-ft-gz', 'fulltext'),
60
    'bzrlib.knit', 'FTAnnotatedToFullText')
61
62
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
63
class ContentFactory(object):
64
    """Abstract interface for insertion and retrieval from a VersionedFile.
65
    
66
    :ivar sha1: None, or the sha1 of the content fulltext.
67
    :ivar storage_kind: The native storage kind of this factory. One of
68
        'mpdiff', 'knit-annotated-ft', 'knit-annotated-delta', 'knit-ft',
69
        'knit-delta', 'fulltext', 'knit-annotated-ft-gz',
70
        'knit-annotated-delta-gz', 'knit-ft-gz', 'knit-delta-gz'.
71
    :ivar key: The key of this content. Each key is a tuple with a single
72
        string in it.
73
    :ivar parents: A tuple of parent keys for self.key. If the object has
74
        no parent information, None (as opposed to () for an empty list of
75
        parents).
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
76
    """
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
77
78
    def __init__(self):
79
        """Create a ContentFactory."""
80
        self.sha1 = None
81
        self.storage_kind = None
82
        self.key = None
83
        self.parents = None
84
85
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
86
class FulltextContentFactory(ContentFactory):
87
    """Static data content factory.
88
89
    This takes a fulltext when created and just returns that during
90
    get_bytes_as('fulltext').
91
    
92
    :ivar sha1: None, or the sha1 of the content fulltext.
93
    :ivar storage_kind: The native storage kind of this factory. Always
94
        'fulltext'.
95
    :ivar key: The key of this content. Each key is a tuple with a single
96
        string in it.
97
    :ivar parents: A tuple of parent keys for self.key. If the object has
98
        no parent information, None (as opposed to () for an empty list of
99
        parents).
100
     """
101
102
    def __init__(self, key, parents, sha1, text):
103
        """Create a ContentFactory."""
104
        self.sha1 = sha1
105
        self.storage_kind = 'fulltext'
106
        self.key = key
107
        self.parents = parents
108
        self._text = text
109
110
    def get_bytes_as(self, storage_kind):
111
        if storage_kind == self.storage_kind:
112
            return self._text
113
        raise errors.UnavailableRepresentation(self.key, storage_kind,
114
            self.storage_kind)
115
116
117
class AbsentContentFactory(ContentFactory):
3350.3.12 by Robert Collins
Generate streams with absent records.
118
    """A placeholder content factory for unavailable texts.
119
    
120
    :ivar sha1: None.
121
    :ivar storage_kind: 'absent'.
122
    :ivar key: The key of this content. Each key is a tuple with a single
123
        string in it.
124
    :ivar parents: None.
125
    """
126
127
    def __init__(self, key):
128
        """Create a ContentFactory."""
129
        self.sha1 = None
130
        self.storage_kind = 'absent'
131
        self.key = key
132
        self.parents = None
133
134
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
135
class AdapterFactory(ContentFactory):
136
    """A content factory to adapt between key prefix's."""
137
138
    def __init__(self, key, parents, adapted):
139
        """Create an adapter factory instance."""
140
        self.key = key
141
        self.parents = parents
142
        self._adapted = adapted
143
144
    def __getattr__(self, attr):
145
        """Return a member from the adapted object."""
146
        if attr in ('key', 'parents'):
147
            return self.__dict__[attr]
148
        else:
149
            return getattr(self._adapted, attr)
150
151
3350.3.14 by Robert Collins
Deprecate VersionedFile.join.
152
def filter_absent(record_stream):
153
    """Adapt a record stream to remove absent records."""
154
    for record in record_stream:
155
        if record.storage_kind != 'absent':
156
            yield record
157
158
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
159
class VersionedFile(object):
160
    """Versioned text file storage.
161
    
162
    A versioned file manages versions of line-based text files,
163
    keeping track of the originating version for each line.
164
165
    To clients the "lines" of the file are represented as a list of
166
    strings. These strings will typically have terminal newline
167
    characters, but this is not required.  In particular files commonly
168
    do not have a newline at the end of the file.
169
170
    Texts are identified by a version-id string.
171
    """
172
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
173
    @staticmethod
2229.2.3 by Aaron Bentley
change reserved_id to is_reserved_id, add check_not_reserved for DRY
174
    def check_not_reserved_id(version_id):
175
        revision.check_not_reserved_id(version_id)
2229.2.1 by Aaron Bentley
Reject reserved ids in versiondfile, tree, branch and repository
176
1563.2.15 by Robert Collins
remove the weavestore assumptions about the number and nature of files it manages.
177
    def copy_to(self, name, transport):
178
        """Copy this versioned file to name on transport."""
179
        raise NotImplementedError(self.copy_to)
1863.1.1 by John Arbash Meinel
Allow Versioned files to do caching if explicitly asked, and implement for Knit
180
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
181
    def get_record_stream(self, versions, ordering, include_delta_closure):
182
        """Get a stream of records for versions.
183
184
        :param versions: The versions to include. Each version is a tuple
185
            (version,).
186
        :param ordering: Either 'unordered' or 'topological'. A topologically
187
            sorted stream has compression parents strictly before their
188
            children.
189
        :param include_delta_closure: If True then the closure across any
3350.3.22 by Robert Collins
Review feedback.
190
            compression parents will be included (in the data content of the
191
            stream, not in the emitted records). This guarantees that
192
            'fulltext' can be used successfully on every record.
3350.3.3 by Robert Collins
Functional get_record_stream interface tests covering full interface.
193
        :return: An iterator of ContentFactory objects, each of which is only
194
            valid until the iterator is advanced.
195
        """
196
        raise NotImplementedError(self.get_record_stream)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
197
198
    def has_version(self, version_id):
199
        """Returns whether version is present."""
200
        raise NotImplementedError(self.has_version)
201
3350.3.8 by Robert Collins
Basic stream insertion, no fast path yet for knit to knit.
202
    def insert_record_stream(self, stream):
203
        """Insert a record stream into this versioned file.
204
205
        :param stream: A stream of records to insert. 
206
        :return: None
207
        :seealso VersionedFile.get_record_stream:
208
        """
209
        raise NotImplementedError
210
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
211
    def add_lines(self, version_id, parents, lines, parent_texts=None,
2805.6.7 by Robert Collins
Review feedback.
212
        left_matching_blocks=None, nostore_sha=None, random_id=False,
213
        check_content=True):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
214
        """Add a single text on top of the versioned file.
215
216
        Must raise RevisionAlreadyPresent if the new version is
217
        already present in file history.
218
219
        Must raise RevisionNotPresent if any of the given parents are
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
220
        not present in file history.
2805.6.3 by Robert Collins
* The ``VersionedFile`` interface no longer protects against misuse when
221
222
        :param lines: A list of lines. Each line must be a bytestring. And all
223
            of them except the last must be terminated with \n and contain no
224
            other \n's. The last line may either contain no \n's or a single
225
            terminated \n. If the lines list does meet this constraint the add
226
            routine may error or may succeed - but you will be unable to read
227
            the data back accurately. (Checking the lines have been split
2805.6.7 by Robert Collins
Review feedback.
228
            correctly is expensive and extremely unlikely to catch bugs so it
229
            is not done at runtime unless check_content is True.)
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
230
        :param parent_texts: An optional dictionary containing the opaque 
2805.6.3 by Robert Collins
* The ``VersionedFile`` interface no longer protects against misuse when
231
            representations of some or all of the parents of version_id to
232
            allow delta optimisations.  VERY IMPORTANT: the texts must be those
233
            returned by add_lines or data corruption can be caused.
2520.4.148 by Aaron Bentley
Updates from review
234
        :param left_matching_blocks: a hint about which areas are common
235
            between the text and its left-hand-parent.  The format is
236
            the SequenceMatcher.get_matching_blocks format.
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
237
        :param nostore_sha: Raise ExistingContent and do not add the lines to
238
            the versioned file if the digest of the lines matches this.
2805.6.4 by Robert Collins
Don't check for existing versions when adding texts with random revision ids.
239
        :param random_id: If True a random id has been selected rather than
240
            an id determined by some deterministic process such as a converter
241
            from a foreign VCS. When True the backend may choose not to check
242
            for uniqueness of the resulting key within the versioned file, so
243
            this should only be done when the result is expected to be unique
244
            anyway.
2805.6.7 by Robert Collins
Review feedback.
245
        :param check_content: If True, the lines supplied are verified to be
246
            bytestrings that are correctly formed lines.
2776.1.1 by Robert Collins
* The ``add_lines`` methods on ``VersionedFile`` implementations has changed
247
        :return: The text sha1, the number of bytes in the text, and an opaque
248
                 representation of the inserted version which can be provided
249
                 back to future add_lines calls in the parent_texts dictionary.
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
250
        """
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
251
        self._check_write_ok()
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
252
        return self._add_lines(version_id, parents, lines, parent_texts,
2805.6.7 by Robert Collins
Review feedback.
253
            left_matching_blocks, nostore_sha, random_id, check_content)
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
254
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
255
    def _add_lines(self, version_id, parents, lines, parent_texts,
2805.6.7 by Robert Collins
Review feedback.
256
        left_matching_blocks, nostore_sha, random_id, check_content):
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
257
        """Helper to do the class specific add_lines."""
1563.2.4 by Robert Collins
First cut at including the knit implementation of versioned_file.
258
        raise NotImplementedError(self.add_lines)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
259
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
260
    def add_lines_with_ghosts(self, version_id, parents, lines,
2805.6.7 by Robert Collins
Review feedback.
261
        parent_texts=None, nostore_sha=None, random_id=False,
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
262
        check_content=True, left_matching_blocks=None):
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
263
        """Add lines to the versioned file, allowing ghosts to be present.
264
        
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
265
        This takes the same parameters as add_lines and returns the same.
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
266
        """
1594.2.23 by Robert Collins
Test versioned file storage handling of clean/dirty status for accessed versioned files.
267
        self._check_write_ok()
1596.2.32 by Robert Collins
Reduce re-extraction of texts during weave to knit joins by providing a memoisation facility.
268
        return self._add_lines_with_ghosts(version_id, parents, lines,
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
269
            parent_texts, nostore_sha, random_id, check_content, left_matching_blocks)
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
270
2794.1.1 by Robert Collins
Allow knits to be instructed not to add a text based on a sha, for commit.
271
    def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
272
        nostore_sha, random_id, check_content, left_matching_blocks):
1594.2.21 by Robert Collins
Teach versioned files to prevent mutation after finishing.
273
        """Helper to do class specific add_lines_with_ghosts."""
1594.2.8 by Robert Collins
add ghost aware apis to knits.
274
        raise NotImplementedError(self.add_lines_with_ghosts)
275
1563.2.19 by Robert Collins
stub out a check for knits.
276
    def check(self, progress_bar=None):
277
        """Check the versioned file for integrity."""
278
        raise NotImplementedError(self.check)
279
1666.1.6 by Robert Collins
Make knit the default format.
280
    def _check_lines_not_unicode(self, lines):
281
        """Check that lines being added to a versioned file are not unicode."""
282
        for line in lines:
283
            if line.__class__ is not str:
284
                raise errors.BzrBadParameterUnicode("lines")
285
286
    def _check_lines_are_lines(self, lines):
287
        """Check that the lines really are full lines without inline EOL."""
288
        for line in lines:
289
            if '\n' in line[:-1]:
290
                raise errors.BzrBadParameterContainsNewline("lines")
291
2535.3.1 by Andrew Bennetts
Add get_format_signature to VersionedFile
292
    def get_format_signature(self):
293
        """Get a text description of the data encoding in this file.
294
        
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
295
        :since: 0.90
2535.3.1 by Andrew Bennetts
Add get_format_signature to VersionedFile
296
        """
297
        raise NotImplementedError(self.get_format_signature)
298
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
299
    def make_mpdiffs(self, version_ids):
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
300
        """Create multiparent diffs for specified versions."""
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
301
        knit_versions = set()
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
302
        knit_versions.update(version_ids)
303
        parent_map = self.get_parent_map(version_ids)
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
304
        for version_id in version_ids:
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
305
            try:
306
                knit_versions.update(parent_map[version_id])
307
            except KeyError:
3453.3.1 by Daniel Fischer
Raise the right exception in make_mpdiffs (bug #235687)
308
                raise errors.RevisionNotPresent(version_id, self)
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
309
        # We need to filter out ghosts, because we can't diff against them.
310
        knit_versions = set(self.get_parent_map(knit_versions).keys())
2520.4.90 by Aaron Bentley
Handle \r terminated lines in Weaves properly
311
        lines = dict(zip(knit_versions,
312
            self._get_lf_split_line_list(knit_versions)))
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
313
        diffs = []
314
        for version_id in version_ids:
315
            target = lines[version_id]
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
316
            try:
317
                parents = [lines[p] for p in parent_map[version_id] if p in
318
                    knit_versions]
319
            except KeyError:
3453.3.2 by John Arbash Meinel
Add a test case for the first loop, unable to find a way to trigger the second loop
320
                # I don't know how this could ever trigger.
321
                # parent_map[version_id] was already triggered in the previous
322
                # for loop, and lines[p] has the 'if p in knit_versions' check,
323
                # so we again won't have a KeyError.
3453.3.1 by Daniel Fischer
Raise the right exception in make_mpdiffs (bug #235687)
324
                raise errors.RevisionNotPresent(version_id, self)
2520.4.48 by Aaron Bentley
Support getting blocks from knit deltas with no final EOL
325
            if len(parents) > 0:
326
                left_parent_blocks = self._extract_blocks(version_id,
327
                                                          parents[0], target)
328
            else:
329
                left_parent_blocks = None
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
330
            diffs.append(multiparent.MultiParent.from_lines(target, parents,
331
                         left_parent_blocks))
332
        return diffs
333
2520.4.48 by Aaron Bentley
Support getting blocks from knit deltas with no final EOL
334
    def _extract_blocks(self, version_id, source, target):
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
335
        return None
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
336
2520.4.61 by Aaron Bentley
Do bulk insertion of records
337
    def add_mpdiffs(self, records):
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
338
        """Add mpdiffs to this VersionedFile.
2520.4.126 by Aaron Bentley
Add more docs
339
340
        Records should be iterables of version, parents, expected_sha1,
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
341
        mpdiff. mpdiff should be a MultiParent instance.
2520.4.126 by Aaron Bentley
Add more docs
342
        """
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
343
        # Does this need to call self._check_write_ok()? (IanC 20070919)
2520.4.61 by Aaron Bentley
Do bulk insertion of records
344
        vf_parents = {}
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
345
        mpvf = multiparent.MultiMemoryVersionedFile()
346
        versions = []
347
        for version, parent_ids, expected_sha1, mpdiff in records:
348
            versions.append(version)
349
            mpvf.add_diff(mpdiff, version, parent_ids)
350
        needed_parents = set()
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
351
        for version, parent_ids, expected_sha1, mpdiff in records:
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
352
            needed_parents.update(p for p in parent_ids
353
                                  if not mpvf.has_version(p))
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
354
        present_parents = set(self.get_parent_map(needed_parents).keys())
355
        for parent_id, lines in zip(present_parents,
356
                                 self._get_lf_split_line_list(present_parents)):
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
357
            mpvf.add_version(lines, parent_id, [])
358
        for (version, parent_ids, expected_sha1, mpdiff), lines in\
359
            zip(records, mpvf.get_line_list(versions)):
360
            if len(parent_ids) == 1:
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
361
                left_matching_blocks = list(mpdiff.get_matching_blocks(0,
2520.4.141 by Aaron Bentley
More batch operations adding mpdiffs
362
                    mpvf.get_diff(parent_ids[0]).num_lines()))
2520.4.140 by Aaron Bentley
Use matching blocks from mpdiff for knit delta creation
363
            else:
364
                left_matching_blocks = None
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
365
            try:
366
                _, _, version_text = self.add_lines_with_ghosts(version,
367
                    parent_ids, lines, vf_parents,
368
                    left_matching_blocks=left_matching_blocks)
369
            except NotImplementedError:
370
                # The vf can't handle ghosts, so add lines normally, which will
371
                # (reasonably) fail if there are ghosts in the data.
372
                _, _, version_text = self.add_lines(version,
373
                    parent_ids, lines, vf_parents,
374
                    left_matching_blocks=left_matching_blocks)
2520.4.61 by Aaron Bentley
Do bulk insertion of records
375
            vf_parents[version] = version_text
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
376
        sha1s = self.get_sha1s(versions)
377
        for version, parent_ids, expected_sha1, mpdiff in records:
378
            if expected_sha1 != sha1s[version]:
2520.4.71 by Aaron Bentley
Update test to accept VersionedFileInvalidChecksum instead of TestamentMismatch
379
                raise errors.VersionedFileInvalidChecksum(version)
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
380
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
381
    def get_text(self, version_id):
382
        """Return version contents as a text string.
383
384
        Raises RevisionNotPresent if version is not present in
385
        file history.
386
        """
387
        return ''.join(self.get_lines(version_id))
388
    get_string = get_text
389
1756.2.1 by Aaron Bentley
Implement get_texts
390
    def get_texts(self, version_ids):
391
        """Return the texts of listed versions as a list of strings.
392
393
        Raises RevisionNotPresent if version is not present in
394
        file history.
395
        """
396
        return [''.join(self.get_lines(v)) for v in version_ids]
397
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
398
    def get_lines(self, version_id):
399
        """Return version contents as a sequence of lines.
400
401
        Raises RevisionNotPresent if version is not present in
402
        file history.
403
        """
404
        raise NotImplementedError(self.get_lines)
405
2520.4.90 by Aaron Bentley
Handle \r terminated lines in Weaves properly
406
    def _get_lf_split_line_list(self, version_ids):
407
        return [StringIO(t).readlines() for t in self.get_texts(version_ids)]
2520.4.3 by Aaron Bentley
Implement plain strategy for extracting and installing multiparent diffs
408
2530.1.1 by Aaron Bentley
Make topological sorting optional for get_ancestry
409
    def get_ancestry(self, version_ids, topo_sorted=True):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
410
        """Return a list of all ancestors of given version(s). This
411
        will not include the null revision.
412
2490.2.32 by Aaron Bentley
Merge of not-sorting-ancestry branch
413
        This list will not be topologically sorted if topo_sorted=False is
414
        passed.
2530.1.1 by Aaron Bentley
Make topological sorting optional for get_ancestry
415
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
416
        Must raise RevisionNotPresent if any of the given versions are
417
        not present in file history."""
418
        if isinstance(version_ids, basestring):
419
            version_ids = [version_ids]
420
        raise NotImplementedError(self.get_ancestry)
421
        
1594.2.8 by Robert Collins
add ghost aware apis to knits.
422
    def get_ancestry_with_ghosts(self, version_ids):
423
        """Return a list of all ancestors of given version(s). This
424
        will not include the null revision.
425
426
        Must raise RevisionNotPresent if any of the given versions are
427
        not present in file history.
428
        
429
        Ghosts that are known about will be included in ancestry list,
430
        but are not explicitly marked.
431
        """
432
        raise NotImplementedError(self.get_ancestry_with_ghosts)
3316.2.7 by Robert Collins
Actually deprecated VersionedFile.get_graph.
433
    
3287.5.1 by Robert Collins
Add VersionedFile.get_parent_map.
434
    def get_parent_map(self, version_ids):
435
        """Get a map of the parents of version_ids.
436
437
        :param version_ids: The version ids to look up parents for.
438
        :return: A mapping from version id to parents.
439
        """
440
        raise NotImplementedError(self.get_parent_map)
441
1594.2.8 by Robert Collins
add ghost aware apis to knits.
442
    def get_parents_with_ghosts(self, version_id):
443
        """Return version names for parents of version_id.
444
445
        Will raise RevisionNotPresent if version_id is not present
446
        in the history.
447
448
        Ghosts that are known about will be included in the parent list,
449
        but are not explicitly marked.
450
        """
3287.5.1 by Robert Collins
Add VersionedFile.get_parent_map.
451
        try:
452
            return list(self.get_parent_map([version_id])[version_id])
453
        except KeyError:
454
            raise errors.RevisionNotPresent(version_id, self)
1594.2.8 by Robert Collins
add ghost aware apis to knits.
455
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
456
    def annotate(self, version_id):
3316.2.13 by Robert Collins
* ``VersionedFile.annotate_iter`` is deprecated. While in principal this
457
        """Return a list of (version-id, line) tuples for version_id.
458
459
        :raise RevisionNotPresent: If the given version is
460
        not present in file history.
461
        """
462
        raise NotImplementedError(self.annotate)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
463
2975.3.2 by Robert Collins
Review feedback - document the API change and improve readability in pack's _do_copy_nodes.
464
    def iter_lines_added_or_present_in_versions(self, version_ids=None,
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
465
                                                pb=None):
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
466
        """Iterate over the lines in the versioned file from version_ids.
467
2975.3.2 by Robert Collins
Review feedback - document the API change and improve readability in pack's _do_copy_nodes.
468
        This may return lines from other versions. Each item the returned
469
        iterator yields is a tuple of a line and a text version that that line
470
        is present in (not introduced in).
471
472
        Ordering of results is in whatever order is most suitable for the
473
        underlying storage format.
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
474
2039.1.1 by Aaron Bentley
Clean up progress properly when interrupted during fetch (#54000)
475
        If a progress bar is supplied, it may be used to indicate progress.
476
        The caller is responsible for cleaning up progress bars (because this
477
        is an iterator).
478
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
479
        NOTES: Lines are normalised: they will all have \n terminators.
480
               Lines are returned in arbitrary order.
2975.3.2 by Robert Collins
Review feedback - document the API change and improve readability in pack's _do_copy_nodes.
481
482
        :return: An iterator over (line, version_id).
1594.2.6 by Robert Collins
Introduce a api specifically for looking at lines in some versions of the inventory, for fileid_involved.
483
        """
484
        raise NotImplementedError(self.iter_lines_added_or_present_in_versions)
485
1551.6.15 by Aaron Bentley
Moved plan_merge into Weave
486
    def plan_merge(self, ver_a, ver_b):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
487
        """Return pseudo-annotation indicating how the two versions merge.
488
489
        This is computed between versions a and b and their common
490
        base.
491
492
        Weave lines present in none of them are skipped entirely.
1664.2.2 by Aaron Bentley
Added legend for plan-merge output
493
494
        Legend:
495
        killed-base Dead in base revision
496
        killed-both Killed in each revision
497
        killed-a    Killed in a
498
        killed-b    Killed in b
499
        unchanged   Alive in both a and b (possibly created in both)
500
        new-a       Created in a
501
        new-b       Created in b
1664.2.5 by Aaron Bentley
Update plan-merge legend
502
        ghost-a     Killed in a, unborn in b    
503
        ghost-b     Killed in b, unborn in a
1664.2.2 by Aaron Bentley
Added legend for plan-merge output
504
        irrelevant  Not in either revision
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
505
        """
1551.6.15 by Aaron Bentley
Moved plan_merge into Weave
506
        raise NotImplementedError(VersionedFile.plan_merge)
507
        
1996.3.7 by John Arbash Meinel
lazy import versionedfile, late-load bzrlib.merge
508
    def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
1551.6.14 by Aaron Bentley
Tweaks from merge review
509
                    b_marker=TextMerge.B_MARKER):
1551.6.12 by Aaron Bentley
Indicate conflicts from merge_lines, insead of guessing
510
        return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
511
1664.2.7 by Aaron Bentley
Merge bzr.dev
512
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
513
class RecordingVersionedFilesDecorator(object):
514
    """A minimal versioned files that records calls made on it.
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
515
    
516
    Only enough methods have been added to support tests using it to date.
517
518
    :ivar calls: A list of the calls made; can be reset at any time by
519
        assigning [] to it.
520
    """
521
522
    def __init__(self, backing_vf):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
523
        """Create a RecordingVersionedFileDsecorator decorating backing_vf.
3350.3.4 by Robert Collins
Finish adapters for annotated knits to unannotated knits and full texts.
524
        
525
        :param backing_vf: The versioned file to answer all methods.
526
        """
527
        self._backing_vf = backing_vf
528
        self.calls = []
529
3350.8.2 by Robert Collins
stacked get_parent_map.
530
    def add_lines(self, key, parents, lines, parent_texts=None,
531
        left_matching_blocks=None, nostore_sha=None, random_id=False,
532
        check_content=True):
533
        self.calls.append(("add_lines", key, parents, lines, parent_texts,
534
            left_matching_blocks, nostore_sha, random_id, check_content))
535
        return self._backing_vf.add_lines(key, parents, lines, parent_texts,
536
            left_matching_blocks, nostore_sha, random_id, check_content)
537
3517.4.19 by Martin Pool
Update test for knit.check() to expect it to recurse into fallback vfs
538
    def check(self):
539
        self._backing_vf.check()
540
3350.8.2 by Robert Collins
stacked get_parent_map.
541
    def get_parent_map(self, keys):
542
        self.calls.append(("get_parent_map", copy(keys)))
543
        return self._backing_vf.get_parent_map(keys)
544
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
545
    def get_record_stream(self, keys, sort_order, include_delta_closure):
3350.8.7 by Robert Collins
get_record_stream for fulltexts working (but note extreme memory use!).
546
        self.calls.append(("get_record_stream", list(keys), sort_order,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
547
            include_delta_closure))
548
        return self._backing_vf.get_record_stream(keys, sort_order,
549
            include_delta_closure)
550
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
551
    def get_sha1s(self, keys):
552
        self.calls.append(("get_sha1s", copy(keys)))
553
        return self._backing_vf.get_sha1s(keys)
554
3350.8.5 by Robert Collins
Iter_lines_added_or_present_in_keys stacks.
555
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
556
        self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
3350.8.14 by Robert Collins
Review feedback.
557
        return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
3350.8.5 by Robert Collins
Iter_lines_added_or_present_in_keys stacks.
558
3350.8.4 by Robert Collins
Vf.keys() stacking support.
559
    def keys(self):
560
        self.calls.append(("keys",))
561
        return self._backing_vf.keys()
562
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
563
564
class KeyMapper(object):
3350.6.10 by Martin Pool
VersionedFiles review cleanups
565
    """KeyMappers map between keys and underlying partitioned storage."""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
566
567
    def map(self, key):
568
        """Map key to an underlying storage identifier.
569
570
        :param key: A key tuple e.g. ('file-id', 'revision-id').
571
        :return: An underlying storage identifier, specific to the partitioning
572
            mechanism.
573
        """
574
        raise NotImplementedError(self.map)
575
576
    def unmap(self, partition_id):
577
        """Map a partitioned storage id back to a key prefix.
578
        
579
        :param partition_id: The underlying partition id.
3350.6.10 by Martin Pool
VersionedFiles review cleanups
580
        :return: As much of a key (or prefix) as is derivable from the partition
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
581
            id.
582
        """
583
        raise NotImplementedError(self.unmap)
584
585
586
class ConstantMapper(KeyMapper):
587
    """A key mapper that maps to a constant result."""
588
589
    def __init__(self, result):
590
        """Create a ConstantMapper which will return result for all maps."""
591
        self._result = result
592
593
    def map(self, key):
594
        """See KeyMapper.map()."""
595
        return self._result
596
597
598
class URLEscapeMapper(KeyMapper):
599
    """Base class for use with transport backed storage.
600
601
    This provides a map and unmap wrapper that respectively url escape and
602
    unescape their outputs and inputs.
603
    """
604
605
    def map(self, key):
606
        """See KeyMapper.map()."""
607
        return urllib.quote(self._map(key))
608
609
    def unmap(self, partition_id):
610
        """See KeyMapper.unmap()."""
611
        return self._unmap(urllib.unquote(partition_id))
612
613
614
class PrefixMapper(URLEscapeMapper):
615
    """A key mapper that extracts the first component of a key.
616
    
617
    This mapper is for use with a transport based backend.
618
    """
619
620
    def _map(self, key):
621
        """See KeyMapper.map()."""
622
        return key[0]
623
624
    def _unmap(self, partition_id):
625
        """See KeyMapper.unmap()."""
626
        return (partition_id,)
627
628
629
class HashPrefixMapper(URLEscapeMapper):
630
    """A key mapper that combines the first component of a key with a hash.
631
632
    This mapper is for use with a transport based backend.
633
    """
634
635
    def _map(self, key):
636
        """See KeyMapper.map()."""
637
        prefix = self._escape(key[0])
638
        return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
639
640
    def _escape(self, prefix):
641
        """No escaping needed here."""
642
        return prefix
643
644
    def _unmap(self, partition_id):
645
        """See KeyMapper.unmap()."""
646
        return (self._unescape(osutils.basename(partition_id)),)
647
648
    def _unescape(self, basename):
649
        """No unescaping needed for HashPrefixMapper."""
650
        return basename
651
652
653
class HashEscapedPrefixMapper(HashPrefixMapper):
654
    """Combines the escaped first component of a key with a hash.
655
    
656
    This mapper is for use with a transport based backend.
657
    """
658
659
    _safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
660
661
    def _escape(self, prefix):
662
        """Turn a key element into a filesystem safe string.
663
664
        This is similar to a plain urllib.quote, except
665
        it uses specific safe characters, so that it doesn't
666
        have to translate a lot of valid file ids.
667
        """
668
        # @ does not get escaped. This is because it is a valid
669
        # filesystem character we use all the time, and it looks
670
        # a lot better than seeing %40 all the time.
671
        r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
672
             for c in prefix]
673
        return ''.join(r)
674
675
    def _unescape(self, basename):
676
        """Escaped names are easily unescaped by urlutils."""
677
        return urllib.unquote(basename)
678
679
680
def make_versioned_files_factory(versioned_file_factory, mapper):
681
    """Create a ThunkedVersionedFiles factory.
682
683
    This will create a callable which when called creates a
684
    ThunkedVersionedFiles on a transport, using mapper to access individual
685
    versioned files, and versioned_file_factory to create each individual file.
686
    """
687
    def factory(transport):
688
        return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
689
            lambda:True)
690
    return factory
691
692
693
class VersionedFiles(object):
694
    """Storage for many versioned files.
695
696
    This object allows a single keyspace for accessing the history graph and
697
    contents of named bytestrings.
698
699
    Currently no implementation allows the graph of different key prefixes to
700
    intersect, but the API does allow such implementations in the future.
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
701
702
    The keyspace is expressed via simple tuples. Any instance of VersionedFiles
703
    may have a different length key-size, but that size will be constant for
704
    all texts added to or retrieved from it. For instance, bzrlib uses
705
    instances with a key-size of 2 for storing user files in a repository, with
706
    the first element the fileid, and the second the version of that file.
707
708
    The use of tuples allows a single code base to support several different
709
    uses with only the mapping logic changing from instance to instance.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
710
    """
711
712
    def add_lines(self, key, parents, lines, parent_texts=None,
713
        left_matching_blocks=None, nostore_sha=None, random_id=False,
714
        check_content=True):
715
        """Add a text to the store.
716
3735.2.5 by Robert Collins
Teach VersionedFiles how to allocate keys based on content hashes.
717
        :param key: The key tuple of the text to add. If the last element is
718
            None, a CHK string will be generated during the addition.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
719
        :param parents: The parents key tuples of the text to add.
720
        :param lines: A list of lines. Each line must be a bytestring. And all
721
            of them except the last must be terminated with \n and contain no
722
            other \n's. The last line may either contain no \n's or a single
723
            terminating \n. If the lines list does meet this constraint the add
724
            routine may error or may succeed - but you will be unable to read
725
            the data back accurately. (Checking the lines have been split
726
            correctly is expensive and extremely unlikely to catch bugs so it
727
            is not done at runtime unless check_content is True.)
728
        :param parent_texts: An optional dictionary containing the opaque 
729
            representations of some or all of the parents of version_id to
730
            allow delta optimisations.  VERY IMPORTANT: the texts must be those
731
            returned by add_lines or data corruption can be caused.
732
        :param left_matching_blocks: a hint about which areas are common
733
            between the text and its left-hand-parent.  The format is
734
            the SequenceMatcher.get_matching_blocks format.
735
        :param nostore_sha: Raise ExistingContent and do not add the lines to
736
            the versioned file if the digest of the lines matches this.
737
        :param random_id: If True a random id has been selected rather than
738
            an id determined by some deterministic process such as a converter
739
            from a foreign VCS. When True the backend may choose not to check
740
            for uniqueness of the resulting key within the versioned file, so
741
            this should only be done when the result is expected to be unique
742
            anyway.
743
        :param check_content: If True, the lines supplied are verified to be
744
            bytestrings that are correctly formed lines.
745
        :return: The text sha1, the number of bytes in the text, and an opaque
746
                 representation of the inserted version which can be provided
747
                 back to future add_lines calls in the parent_texts dictionary.
748
        """
749
        raise NotImplementedError(self.add_lines)
750
751
    def add_mpdiffs(self, records):
752
        """Add mpdiffs to this VersionedFile.
753
754
        Records should be iterables of version, parents, expected_sha1,
755
        mpdiff. mpdiff should be a MultiParent instance.
756
        """
757
        vf_parents = {}
758
        mpvf = multiparent.MultiMemoryVersionedFile()
759
        versions = []
760
        for version, parent_ids, expected_sha1, mpdiff in records:
761
            versions.append(version)
762
            mpvf.add_diff(mpdiff, version, parent_ids)
763
        needed_parents = set()
764
        for version, parent_ids, expected_sha1, mpdiff in records:
765
            needed_parents.update(p for p in parent_ids
766
                                  if not mpvf.has_version(p))
767
        # It seems likely that adding all the present parents as fulltexts can
768
        # easily exhaust memory.
769
        split_lines = osutils.split_lines
3350.8.11 by Robert Collins
Stacked add_mpdiffs.
770
        for record in self.get_record_stream(needed_parents, 'unordered',
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
771
            True):
3350.8.11 by Robert Collins
Stacked add_mpdiffs.
772
            if record.storage_kind == 'absent':
773
                continue
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
774
            mpvf.add_version(split_lines(record.get_bytes_as('fulltext')),
775
                record.key, [])
776
        for (key, parent_keys, expected_sha1, mpdiff), lines in\
777
            zip(records, mpvf.get_line_list(versions)):
778
            if len(parent_keys) == 1:
779
                left_matching_blocks = list(mpdiff.get_matching_blocks(0,
780
                    mpvf.get_diff(parent_keys[0]).num_lines()))
781
            else:
782
                left_matching_blocks = None
783
            version_sha1, _, version_text = self.add_lines(key,
784
                parent_keys, lines, vf_parents,
785
                left_matching_blocks=left_matching_blocks)
786
            if version_sha1 != expected_sha1:
787
                raise errors.VersionedFileInvalidChecksum(version)
788
            vf_parents[key] = version_text
789
790
    def annotate(self, key):
791
        """Return a list of (version-key, line) tuples for the text of key.
792
793
        :raise RevisionNotPresent: If the key is not present.
794
        """
795
        raise NotImplementedError(self.annotate)
796
797
    def check(self, progress_bar=None):
798
        """Check this object for integrity."""
799
        raise NotImplementedError(self.check)
800
801
    @staticmethod
802
    def check_not_reserved_id(version_id):
803
        revision.check_not_reserved_id(version_id)
804
805
    def _check_lines_not_unicode(self, lines):
806
        """Check that lines being added to a versioned file are not unicode."""
807
        for line in lines:
808
            if line.__class__ is not str:
809
                raise errors.BzrBadParameterUnicode("lines")
810
811
    def _check_lines_are_lines(self, lines):
812
        """Check that the lines really are full lines without inline EOL."""
813
        for line in lines:
814
            if '\n' in line[:-1]:
815
                raise errors.BzrBadParameterContainsNewline("lines")
816
817
    def get_parent_map(self, keys):
818
        """Get a map of the parents of keys.
819
820
        :param keys: The keys to look up parents for.
821
        :return: A mapping from keys to parents. Absent keys are absent from
822
            the mapping.
823
        """
824
        raise NotImplementedError(self.get_parent_map)
825
826
    def get_record_stream(self, keys, ordering, include_delta_closure):
827
        """Get a stream of records for keys.
828
829
        :param keys: The keys to include.
830
        :param ordering: Either 'unordered' or 'topological'. A topologically
831
            sorted stream has compression parents strictly before their
832
            children.
833
        :param include_delta_closure: If True then the closure across any
834
            compression parents will be included (in the opaque data).
835
        :return: An iterator of ContentFactory objects, each of which is only
836
            valid until the iterator is advanced.
837
        """
838
        raise NotImplementedError(self.get_record_stream)
839
840
    def get_sha1s(self, keys):
841
        """Get the sha1's of the texts for the given keys.
842
843
        :param keys: The names of the keys to lookup
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
844
        :return: a dict from key to sha1 digest. Keys of texts which are not
3350.8.14 by Robert Collins
Review feedback.
845
            present in the store are not present in the returned
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
846
            dictionary.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
847
        """
848
        raise NotImplementedError(self.get_sha1s)
849
850
    def insert_record_stream(self, stream):
851
        """Insert a record stream into this container.
852
853
        :param stream: A stream of records to insert. 
854
        :return: None
855
        :seealso VersionedFile.get_record_stream:
856
        """
857
        raise NotImplementedError
858
859
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
860
        """Iterate over the lines in the versioned files from keys.
861
862
        This may return lines from other keys. Each item the returned
863
        iterator yields is a tuple of a line and a text version that that line
864
        is present in (not introduced in).
865
866
        Ordering of results is in whatever order is most suitable for the
867
        underlying storage format.
868
869
        If a progress bar is supplied, it may be used to indicate progress.
870
        The caller is responsible for cleaning up progress bars (because this
871
        is an iterator).
872
873
        NOTES:
874
         * Lines are normalised by the underlying store: they will all have \n
875
           terminators.
876
         * Lines are returned in arbitrary order.
877
878
        :return: An iterator over (line, key).
879
        """
880
        raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
881
882
    def keys(self):
883
        """Return a iterable of the keys for all the contained texts."""
884
        raise NotImplementedError(self.keys)
885
886
    def make_mpdiffs(self, keys):
887
        """Create multiparent diffs for specified keys."""
888
        keys_order = tuple(keys)
889
        keys = frozenset(keys)
890
        knit_keys = set(keys)
891
        parent_map = self.get_parent_map(keys)
892
        for parent_keys in parent_map.itervalues():
893
            if parent_keys:
894
                knit_keys.update(parent_keys)
895
        missing_keys = keys - set(parent_map)
896
        if missing_keys:
3530.3.2 by Robert Collins
Handling frozen set inputs in mpdiff generation when a key is missing
897
            raise errors.RevisionNotPresent(list(missing_keys)[0], self)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
898
        # We need to filter out ghosts, because we can't diff against them.
899
        maybe_ghosts = knit_keys - keys
900
        ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
901
        knit_keys.difference_update(ghosts)
902
        lines = {}
903
        split_lines = osutils.split_lines
904
        for record in self.get_record_stream(knit_keys, 'topological', True):
905
            lines[record.key] = split_lines(record.get_bytes_as('fulltext'))
906
            # line_block_dict = {}
907
            # for parent, blocks in record.extract_line_blocks():
908
            #   line_blocks[parent] = blocks
909
            # line_blocks[record.key] = line_block_dict
910
        diffs = []
911
        for key in keys_order:
912
            target = lines[key]
913
            parents = parent_map[key] or []
914
            # Note that filtering knit_keys can lead to a parent difference
915
            # between the creation and the application of the mpdiff.
916
            parent_lines = [lines[p] for p in parents if p in knit_keys]
917
            if len(parent_lines) > 0:
918
                left_parent_blocks = self._extract_blocks(key, parent_lines[0],
919
                    target)
920
            else:
921
                left_parent_blocks = None
922
            diffs.append(multiparent.MultiParent.from_lines(target,
923
                parent_lines, left_parent_blocks))
924
        return diffs
925
926
    def _extract_blocks(self, version_id, source, target):
927
        return None
928
929
930
class ThunkedVersionedFiles(VersionedFiles):
931
    """Storage for many versioned files thunked onto a 'VersionedFile' class.
932
933
    This object allows a single keyspace for accessing the history graph and
934
    contents of named bytestrings.
935
936
    Currently no implementation allows the graph of different key prefixes to
937
    intersect, but the API does allow such implementations in the future.
938
    """
939
940
    def __init__(self, transport, file_factory, mapper, is_locked):
941
        """Create a ThunkedVersionedFiles."""
942
        self._transport = transport
943
        self._file_factory = file_factory
944
        self._mapper = mapper
945
        self._is_locked = is_locked
946
947
    def add_lines(self, key, parents, lines, parent_texts=None,
948
        left_matching_blocks=None, nostore_sha=None, random_id=False,
949
        check_content=True):
950
        """See VersionedFiles.add_lines()."""
951
        path = self._mapper.map(key)
952
        version_id = key[-1]
953
        parents = [parent[-1] for parent in parents]
954
        vf = self._get_vf(path)
955
        try:
956
            try:
957
                return vf.add_lines_with_ghosts(version_id, parents, lines,
958
                    parent_texts=parent_texts,
959
                    left_matching_blocks=left_matching_blocks,
960
                    nostore_sha=nostore_sha, random_id=random_id,
961
                    check_content=check_content)
962
            except NotImplementedError:
963
                return vf.add_lines(version_id, parents, lines,
964
                    parent_texts=parent_texts,
965
                    left_matching_blocks=left_matching_blocks,
966
                    nostore_sha=nostore_sha, random_id=random_id,
967
                    check_content=check_content)
968
        except errors.NoSuchFile:
969
            # parent directory may be missing, try again.
970
            self._transport.mkdir(osutils.dirname(path))
971
            try:
972
                return vf.add_lines_with_ghosts(version_id, parents, lines,
973
                    parent_texts=parent_texts,
974
                    left_matching_blocks=left_matching_blocks,
975
                    nostore_sha=nostore_sha, random_id=random_id,
976
                    check_content=check_content)
977
            except NotImplementedError:
978
                return vf.add_lines(version_id, parents, lines,
979
                    parent_texts=parent_texts,
980
                    left_matching_blocks=left_matching_blocks,
981
                    nostore_sha=nostore_sha, random_id=random_id,
982
                    check_content=check_content)
983
984
    def annotate(self, key):
985
        """Return a list of (version-key, line) tuples for the text of key.
986
987
        :raise RevisionNotPresent: If the key is not present.
988
        """
989
        prefix = key[:-1]
990
        path = self._mapper.map(prefix)
991
        vf = self._get_vf(path)
992
        origins = vf.annotate(key[-1])
993
        result = []
994
        for origin, line in origins:
995
            result.append((prefix + (origin,), line))
996
        return result
997
998
    def check(self, progress_bar=None):
999
        """See VersionedFiles.check()."""
1000
        for prefix, vf in self._iter_all_components():
1001
            vf.check()
1002
1003
    def get_parent_map(self, keys):
1004
        """Get a map of the parents of keys.
1005
1006
        :param keys: The keys to look up parents for.
1007
        :return: A mapping from keys to parents. Absent keys are absent from
1008
            the mapping.
1009
        """
1010
        prefixes = self._partition_keys(keys)
1011
        result = {}
1012
        for prefix, suffixes in prefixes.items():
1013
            path = self._mapper.map(prefix)
1014
            vf = self._get_vf(path)
1015
            parent_map = vf.get_parent_map(suffixes)
1016
            for key, parents in parent_map.items():
1017
                result[prefix + (key,)] = tuple(
1018
                    prefix + (parent,) for parent in parents)
1019
        return result
1020
1021
    def _get_vf(self, path):
1022
        if not self._is_locked():
1023
            raise errors.ObjectNotLocked(self)
1024
        return self._file_factory(path, self._transport, create=True,
1025
            get_scope=lambda:None)
1026
1027
    def _partition_keys(self, keys):
1028
        """Turn keys into a dict of prefix:suffix_list."""
1029
        result = {}
1030
        for key in keys:
1031
            prefix_keys = result.setdefault(key[:-1], [])
1032
            prefix_keys.append(key[-1])
1033
        return result
1034
1035
    def _get_all_prefixes(self):
1036
        # Identify all key prefixes.
1037
        # XXX: A bit hacky, needs polish.
1038
        if type(self._mapper) == ConstantMapper:
1039
            paths = [self._mapper.map(())]
1040
            prefixes = [()]
1041
        else:
1042
            relpaths = set()
1043
            for quoted_relpath in self._transport.iter_files_recursive():
1044
                path, ext = os.path.splitext(quoted_relpath)
1045
                relpaths.add(path)
1046
            paths = list(relpaths)
1047
            prefixes = [self._mapper.unmap(path) for path in paths]
1048
        return zip(paths, prefixes)
1049
1050
    def get_record_stream(self, keys, ordering, include_delta_closure):
1051
        """See VersionedFiles.get_record_stream()."""
1052
        # Ordering will be taken care of by each partitioned store; group keys
1053
        # by partition.
1054
        keys = sorted(keys)
1055
        for prefix, suffixes, vf in self._iter_keys_vf(keys):
1056
            suffixes = [(suffix,) for suffix in suffixes]
1057
            for record in vf.get_record_stream(suffixes, ordering,
1058
                include_delta_closure):
1059
                if record.parents is not None:
1060
                    record.parents = tuple(
1061
                        prefix + parent for parent in record.parents)
1062
                record.key = prefix + record.key
1063
                yield record
1064
1065
    def _iter_keys_vf(self, keys):
1066
        prefixes = self._partition_keys(keys)
1067
        sha1s = {}
1068
        for prefix, suffixes in prefixes.items():
1069
            path = self._mapper.map(prefix)
1070
            vf = self._get_vf(path)
1071
            yield prefix, suffixes, vf
1072
1073
    def get_sha1s(self, keys):
1074
        """See VersionedFiles.get_sha1s()."""
1075
        sha1s = {}
1076
        for prefix,suffixes, vf in self._iter_keys_vf(keys):
1077
            vf_sha1s = vf.get_sha1s(suffixes)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1078
            for suffix, sha1 in vf_sha1s.iteritems():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1079
                sha1s[prefix + (suffix,)] = sha1
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
1080
        return sha1s
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1081
1082
    def insert_record_stream(self, stream):
1083
        """Insert a record stream into this container.
1084
1085
        :param stream: A stream of records to insert. 
1086
        :return: None
1087
        :seealso VersionedFile.get_record_stream:
1088
        """
1089
        for record in stream:
1090
            prefix = record.key[:-1]
1091
            key = record.key[-1:]
1092
            if record.parents is not None:
1093
                parents = [parent[-1:] for parent in record.parents]
1094
            else:
1095
                parents = None
1096
            thunk_record = AdapterFactory(key, parents, record)
1097
            path = self._mapper.map(prefix)
1098
            # Note that this parses the file many times; we can do better but
1099
            # as this only impacts weaves in terms of performance, it is
1100
            # tolerable.
1101
            vf = self._get_vf(path)
1102
            vf.insert_record_stream([thunk_record])
1103
1104
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1105
        """Iterate over the lines in the versioned files from keys.
1106
1107
        This may return lines from other keys. Each item the returned
1108
        iterator yields is a tuple of a line and a text version that that line
1109
        is present in (not introduced in).
1110
1111
        Ordering of results is in whatever order is most suitable for the
1112
        underlying storage format.
1113
1114
        If a progress bar is supplied, it may be used to indicate progress.
1115
        The caller is responsible for cleaning up progress bars (because this
1116
        is an iterator).
1117
1118
        NOTES:
1119
         * Lines are normalised by the underlying store: they will all have \n
1120
           terminators.
1121
         * Lines are returned in arbitrary order.
1122
1123
        :return: An iterator over (line, key).
1124
        """
1125
        for prefix, suffixes, vf in self._iter_keys_vf(keys):
1126
            for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1127
                yield line, prefix + (version,)
1128
1129
    def _iter_all_components(self):
1130
        for path, prefix in self._get_all_prefixes():
1131
            yield prefix, self._get_vf(path)
1132
1133
    def keys(self):
1134
        """See VersionedFiles.keys()."""
1135
        result = set()
1136
        for prefix, vf in self._iter_all_components():
1137
            for suffix in vf.versions():
1138
                result.add(prefix + (suffix,))
1139
        return result
1140
1141
1142
class _PlanMergeVersionedFile(VersionedFiles):
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1143
    """A VersionedFile for uncommitted and committed texts.
1144
1145
    It is intended to allow merges to be planned with working tree texts.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1146
    It implements only the small part of the VersionedFiles interface used by
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1147
    PlanMerge.  It falls back to multiple versionedfiles for data not stored in
1148
    _PlanMergeVersionedFile itself.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1149
1150
    :ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1151
        queried for missing texts.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1152
    """
1153
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1154
    def __init__(self, file_id):
1155
        """Create a _PlanMergeVersionedFile.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1156
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1157
        :param file_id: Used with _PlanMerge code which is not yet fully
1158
            tuple-keyspace aware.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1159
        """
1160
        self._file_id = file_id
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1161
        # fallback locations
1162
        self.fallback_versionedfiles = []
1163
        # Parents for locally held keys.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1164
        self._parents = {}
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1165
        # line data for locally held keys.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1166
        self._lines = {}
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1167
        # key lookup providers
1168
        self._providers = [DictParentsProvider(self._parents)]
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1169
3062.2.3 by Aaron Bentley
Sync up with bzr.dev API changes
1170
    def plan_merge(self, ver_a, ver_b, base=None):
3062.1.13 by Aaron Bentley
Make _PlanMerge an implementation detail of _PlanMergeVersionedFile
1171
        """See VersionedFile.plan_merge"""
3144.3.7 by Aaron Bentley
Update from review
1172
        from bzrlib.merge import _PlanMerge
3062.2.3 by Aaron Bentley
Sync up with bzr.dev API changes
1173
        if base is None:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1174
            return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1175
        old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1176
        new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
3062.2.3 by Aaron Bentley
Sync up with bzr.dev API changes
1177
        return _PlanMerge._subtract_plans(old_plan, new_plan)
1178
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1179
    def plan_lca_merge(self, ver_a, ver_b, base=None):
3144.3.7 by Aaron Bentley
Update from review
1180
        from bzrlib.merge import _PlanLCAMerge
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1181
        graph = Graph(self)
1182
        new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1183
        if base is None:
1184
            return new_plan
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1185
        old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1186
        return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
3062.1.13 by Aaron Bentley
Make _PlanMerge an implementation detail of _PlanMergeVersionedFile
1187
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1188
    def add_lines(self, key, parents, lines):
1189
        """See VersionedFiles.add_lines
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1190
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1191
        Lines are added locally, not to fallback versionedfiles.  Also, ghosts
1192
        are permitted.  Only reserved ids are permitted.
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1193
        """
3350.6.8 by Martin Pool
Change stray pdb calls to exceptions
1194
        if type(key) is not tuple:
1195
            raise TypeError(key)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1196
        if not revision.is_reserved_id(key[-1]):
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1197
            raise ValueError('Only reserved ids may be used')
1198
        if parents is None:
1199
            raise ValueError('Parents may not be None')
1200
        if lines is None:
1201
            raise ValueError('Lines may not be None')
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1202
        self._parents[key] = tuple(parents)
1203
        self._lines[key] = lines
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1204
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1205
    def get_record_stream(self, keys, ordering, include_delta_closure):
1206
        pending = set(keys)
1207
        for key in keys:
1208
            if key in self._lines:
1209
                lines = self._lines[key]
1210
                parents = self._parents[key]
1211
                pending.remove(key)
1212
                yield FulltextContentFactory(key, parents, None,
1213
                    ''.join(lines))
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1214
        for versionedfile in self.fallback_versionedfiles:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1215
            for record in versionedfile.get_record_stream(
1216
                pending, 'unordered', True):
1217
                if record.storage_kind == 'absent':
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1218
                    continue
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1219
                else:
1220
                    pending.remove(record.key)
1221
                    yield record
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
1222
            if not pending:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1223
                return
1224
        # report absent entries
1225
        for key in pending:
1226
            yield AbsentContentFactory(key)
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1227
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1228
    def get_parent_map(self, keys):
1229
        """See VersionedFiles.get_parent_map"""
1230
        # We create a new provider because a fallback may have been added.
1231
        # If we make fallbacks private we can update a stack list and avoid
1232
        # object creation thrashing.
3350.6.6 by Robert Collins
Fix test_plan_file_merge
1233
        keys = set(keys)
1234
        result = {}
1235
        if revision.NULL_REVISION in keys:
1236
            keys.remove(revision.NULL_REVISION)
1237
            result[revision.NULL_REVISION] = ()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
1238
        self._providers = self._providers[:1] + self.fallback_versionedfiles
3350.6.6 by Robert Collins
Fix test_plan_file_merge
1239
        result.update(
1240
            _StackedParentsProvider(self._providers).get_parent_map(keys))
3350.6.5 by Robert Collins
Update to bzr.dev.
1241
        for key, parents in result.iteritems():
1242
            if parents == ():
1243
                result[key] = (revision.NULL_REVISION,)
3287.5.2 by Robert Collins
Deprecate VersionedFile.get_parents, breaking pulling from a ghost containing knit or pack repository to weaves, which improves correctness and allows simplification of core code.
1244
        return result
3144.3.1 by Aaron Bentley
Implement LCA merge, with problematic conflict markers
1245
3062.1.9 by Aaron Bentley
Move PlanMerge into merge and _PlanMergeVersionedFile into versionedfile
1246
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1247
class PlanWeaveMerge(TextMerge):
1551.6.13 by Aaron Bentley
Cleanup
1248
    """Weave merge that takes a plan as its input.
1249
    
1551.6.14 by Aaron Bentley
Tweaks from merge review
1250
    This exists so that VersionedFile.plan_merge is implementable.
1251
    Most callers will want to use WeaveMerge instead.
1551.6.13 by Aaron Bentley
Cleanup
1252
    """
1253
1551.6.14 by Aaron Bentley
Tweaks from merge review
1254
    def __init__(self, plan, a_marker=TextMerge.A_MARKER,
1255
                 b_marker=TextMerge.B_MARKER):
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1256
        TextMerge.__init__(self, a_marker, b_marker)
1257
        self.plan = plan
1258
1551.6.7 by Aaron Bentley
Implemented two-way merge, refactored weave merge
1259
    def _merge_struct(self):
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1260
        lines_a = []
1261
        lines_b = []
1262
        ch_a = ch_b = False
1664.2.8 by Aaron Bentley
Fix WeaveMerge when plan doesn't end with unchanged lines
1263
1264
        def outstanding_struct():
1265
            if not lines_a and not lines_b:
1266
                return
1267
            elif ch_a and not ch_b:
1268
                # one-sided change:
1269
                yield(lines_a,)
1270
            elif ch_b and not ch_a:
1271
                yield (lines_b,)
1272
            elif lines_a == lines_b:
1273
                yield(lines_a,)
1274
            else:
1275
                yield (lines_a, lines_b)
1551.6.13 by Aaron Bentley
Cleanup
1276
       
1616.1.18 by Martin Pool
(weave-merge) don't treat killed-both lines as points of agreement;
1277
        # We previously considered either 'unchanged' or 'killed-both' lines
1278
        # to be possible places to resynchronize.  However, assuming agreement
1759.2.1 by Jelmer Vernooij
Fix some types (found using aspell).
1279
        # on killed-both lines may be too aggressive. -- mbp 20060324
1551.6.7 by Aaron Bentley
Implemented two-way merge, refactored weave merge
1280
        for state, line in self.plan:
1616.1.18 by Martin Pool
(weave-merge) don't treat killed-both lines as points of agreement;
1281
            if state == 'unchanged':
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1282
                # resync and flush queued conflicts changes if any
1664.2.8 by Aaron Bentley
Fix WeaveMerge when plan doesn't end with unchanged lines
1283
                for struct in outstanding_struct():
1284
                    yield struct
1551.6.11 by Aaron Bentley
Switched TextMerge_lines to work on a list
1285
                lines_a = []
1286
                lines_b = []
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1287
                ch_a = ch_b = False
1288
                
1289
            if state == 'unchanged':
1290
                if line:
1551.6.5 by Aaron Bentley
Got weave merge producing structural output
1291
                    yield ([line],)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1292
            elif state == 'killed-a':
1293
                ch_a = True
1294
                lines_b.append(line)
1295
            elif state == 'killed-b':
1296
                ch_b = True
1297
                lines_a.append(line)
1298
            elif state == 'new-a':
1299
                ch_a = True
1300
                lines_a.append(line)
1301
            elif state == 'new-b':
1302
                ch_b = True
1303
                lines_b.append(line)
3144.3.2 by Aaron Bentley
Get conflict handling working
1304
            elif state == 'conflicted-a':
1305
                ch_b = ch_a = True
1306
                lines_a.append(line)
1307
            elif state == 'conflicted-b':
1308
                ch_b = ch_a = True
1309
                lines_b.append(line)
1563.2.1 by Robert Collins
Merge in a variation of the versionedfile api from versioned-file.
1310
            else:
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1311
                if state not in ('irrelevant', 'ghost-a', 'ghost-b',
1312
                        'killed-base', 'killed-both'):
1313
                    raise AssertionError(state)
1664.2.8 by Aaron Bentley
Fix WeaveMerge when plan doesn't end with unchanged lines
1314
        for struct in outstanding_struct():
1315
            yield struct
1563.2.12 by Robert Collins
Checkpointing: created InterObject to factor out common inter object worker code, added InterVersionedFile and tests to allow making join work between any versionedfile.
1316
1664.2.14 by Aaron Bentley
spacing fix
1317
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1318
class WeaveMerge(PlanWeaveMerge):
2831.7.1 by Ian Clatworthy
versionedfile.py code cleanups
1319
    """Weave merge that takes a VersionedFile and two versions as its input."""
1551.6.13 by Aaron Bentley
Cleanup
1320
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1321
    def __init__(self, versionedfile, ver_a, ver_b, 
1551.6.14 by Aaron Bentley
Tweaks from merge review
1322
        a_marker=PlanWeaveMerge.A_MARKER, b_marker=PlanWeaveMerge.B_MARKER):
1551.6.15 by Aaron Bentley
Moved plan_merge into Weave
1323
        plan = versionedfile.plan_merge(ver_a, ver_b)
1551.6.10 by Aaron Bentley
Renamed WeaveMerge to PlanMerge, added plan method, created planless WeaveMerge
1324
        PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
1325
1326
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1327
class VirtualVersionedFiles(VersionedFiles):
1328
    """Dummy implementation for VersionedFiles that uses other functions for 
1329
    obtaining fulltexts and parent maps.
1330
1331
    This is always on the bottom of the stack and uses string keys 
1332
    (rather than tuples) internally.
1333
    """
1334
1335
    def __init__(self, get_parent_map, get_lines):
1336
        """Create a VirtualVersionedFiles.
1337
1338
        :param get_parent_map: Same signature as Repository.get_parent_map.
1339
        :param get_lines: Should return lines for specified key or None if 
1340
                          not available.
1341
        """
1342
        super(VirtualVersionedFiles, self).__init__()
1343
        self._get_parent_map = get_parent_map
1344
        self._get_lines = get_lines
1345
        
1346
    def check(self, progressbar=None):
1347
        """See VersionedFiles.check.
1348
1349
        :note: Always returns True for VirtualVersionedFiles.
1350
        """
1351
        return True
1352
1353
    def add_mpdiffs(self, records):
1354
        """See VersionedFiles.mpdiffs.
1355
1356
        :note: Not implemented for VirtualVersionedFiles.
1357
        """
1358
        raise NotImplementedError(self.add_mpdiffs)
1359
1360
    def get_parent_map(self, keys):
1361
        """See VersionedFiles.get_parent_map."""
3518.1.2 by Jelmer Vernooij
Fix some stylistic issues pointed out by Ian.
1362
        return dict([((k,), tuple([(p,) for p in v]))
1363
            for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1364
1365
    def get_sha1s(self, keys):
1366
        """See VersionedFiles.get_sha1s."""
1367
        ret = {}
1368
        for (k,) in keys:
1369
            lines = self._get_lines(k)
1370
            if lines is not None:
3518.1.2 by Jelmer Vernooij
Fix some stylistic issues pointed out by Ian.
1371
                if not isinstance(lines, list):
1372
                    raise AssertionError
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1373
                ret[(k,)] = osutils.sha_strings(lines)
1374
        return ret
1375
1376
    def get_record_stream(self, keys, ordering, include_delta_closure):
1377
        """See VersionedFiles.get_record_stream."""
1378
        for (k,) in list(keys):
1379
            lines = self._get_lines(k)
1380
            if lines is not None:
3518.1.2 by Jelmer Vernooij
Fix some stylistic issues pointed out by Ian.
1381
                if not isinstance(lines, list):
1382
                    raise AssertionError
3518.1.1 by Jelmer Vernooij
Add VirtualVersionedFiles class.
1383
                yield FulltextContentFactory((k,), None, 
1384
                        sha1=osutils.sha_strings(lines),
1385
                        text=''.join(lines))
1386
            else:
1387
                yield AbsentContentFactory((k,))
1388
1389
1390