/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
1
# Copyright (C) 2007-2010 Canonical Ltd
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
16
6379.6.3 by Jelmer Vernooij
Use absolute_import.
17
from __future__ import absolute_import
18
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
19
import bz2
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
20
import re
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
21
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
22
from ... import (
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
23
    bencode,
2520.4.34 by Aaron Bentley
Add signature support
24
    errors,
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
25
    iterablefile,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
26
    lru_cache,
2520.4.13 by Aaron Bentley
Use real container implementation
27
    multiparent,
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
28
    osutils,
2520.4.13 by Aaron Bentley
Use real container implementation
29
    pack,
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
30
    revision as _mod_revision,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
31
    serializer,
2520.4.45 by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory
32
    trace,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
33
    ui,
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
34
    versionedfile as _mod_versionedfile,
2520.4.13 by Aaron Bentley
Use real container implementation
35
    )
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
36
from ...bundle import bundle_data, serializer as bundle_serializer
37
from ...i18n import ngettext
38
from ...sixish import (
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
39
    BytesIO,
40
    )
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
41
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
42
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
43
class _MPDiffInventoryGenerator(_mod_versionedfile._MPDiffGenerator):
44
    """Generate Inventory diffs serialized inventories."""
45
46
    def __init__(self, repo, inventory_keys):
47
        super(_MPDiffInventoryGenerator, self).__init__(repo.inventories,
48
            inventory_keys)
49
        self.repo = repo
50
        self.sha1s = {}
51
52
    def iter_diffs(self):
53
        """Compute the diffs one at a time."""
54
        # This is instead of compute_diffs() since we guarantee our ordering of
55
        # inventories, we don't have to do any buffering
56
        self._find_needed_keys()
57
        # We actually use a slightly different ordering. We grab all of the
58
        # parents first, and then grab the ordered requests.
59
        needed_ids = [k[-1] for k in self.present_parents]
60
        needed_ids.extend([k[-1] for k in self.ordered_keys])
61
        inv_to_str = self.repo._serializer.write_inventory_to_string
62
        for inv in self.repo.iter_inventories(needed_ids):
63
            revision_id = inv.revision_id
64
            key = (revision_id,)
65
            if key in self.present_parents:
66
                # Not a key we will transmit, which is a shame, since because
67
                # of that bundles don't work with stacked branches
68
                parent_ids = None
69
            else:
70
                parent_ids = [k[-1] for k in self.parent_map[key]]
71
            as_bytes = inv_to_str(inv)
72
            self._process_one_record(key, (as_bytes,))
73
            if parent_ids is None:
74
                continue
75
            diff = self.diffs.pop(key)
76
            sha1 = osutils.sha_string(as_bytes)
77
            yield revision_id, parent_ids, sha1, diff
78
79
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
80
class BundleWriter(object):
2520.4.118 by Aaron Bentley
Add docs
81
    """Writer for bundle-format files.
82
83
    This serves roughly the same purpose as ContainerReader, but acts as a
84
    layer on top of it.
85
2520.4.123 by Aaron Bentley
Cleanup of bundle code
86
    Provides ways of writing the specific record types supported this bundle
2520.4.118 by Aaron Bentley
Add docs
87
    format.
88
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
89
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
90
    def __init__(self, fileobj):
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
91
        self._container = pack.ContainerWriter(self._write_encoded)
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
92
        self._fileobj = fileobj
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
93
        self._compressor = bz2.BZ2Compressor()
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
94
2520.4.118 by Aaron Bentley
Add docs
95
    def _write_encoded(self, bytes):
96
        """Write bzip2-encoded bytes to the file"""
97
        self._fileobj.write(self._compressor.compress(bytes))
98
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
99
    def begin(self):
2520.4.118 by Aaron Bentley
Add docs
100
        """Start writing the bundle"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
101
        self._fileobj.write(bundle_serializer._get_bundle_header(
102
            bundle_serializer.v4_string))
2520.4.24 by Aaron Bentley
Move heading writing above container beginning
103
        self._fileobj.write('#\n')
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
104
        self._container.begin()
105
106
    def end(self):
2520.4.118 by Aaron Bentley
Add docs
107
        """Finish writing the bundle"""
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
108
        self._container.end()
2520.4.76 by Aaron Bentley
Move base64-encoding into merge directives
109
        self._fileobj.write(self._compressor.flush())
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
110
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
111
    def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
112
                               revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
113
        """Add a record for a multi-parent diff
114
115
        :mp_bytes: A multi-parent diff, as a bytestring
2520.4.123 by Aaron Bentley
Cleanup of bundle code
116
        :sha1: The sha1 hash of the fulltext
2520.4.118 by Aaron Bentley
Add docs
117
        :parents: a list of revision-ids of the parents
118
        :repo_kind: The kind of object in the repository.  May be 'file' or
119
            'inventory'
120
        :revision_id: The revision id of the mpdiff being added.
121
        :file_id: The file-id of the file, or None for inventories.
122
        """
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
123
        metadata = {'parents': parents,
124
                    'storage_kind': 'mpdiff',
125
                    'sha1': sha1}
126
        self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
127
2520.4.123 by Aaron Bentley
Cleanup of bundle code
128
    def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
2520.4.118 by Aaron Bentley
Add docs
129
        """Add a record for a fulltext
130
131
        :bytes: The fulltext, as a bytestring
132
        :parents: a list of revision-ids of the parents
133
        :repo_kind: The kind of object in the repository.  May be 'revision' or
134
            'signature'
135
        :revision_id: The revision id of the fulltext being added.
136
        """
137
        metadata = {'parents': parents,
2520.5.3 by Aaron Bentley
fix sha1 in bundle format 4
138
                    'storage_kind': 'mpdiff'}
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
139
        self._add_record(bytes, {'parents': parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
140
            'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
141
2520.4.95 by Aaron Bentley
Add support for header/info records
142
    def add_info_record(self, **kwargs):
2520.4.118 by Aaron Bentley
Add docs
143
        """Add an info record to the bundle
144
145
        Any parameters may be supplied, except 'self' and 'storage_kind'.
146
        Values must be lists, strings, integers, dicts, or a combination.
147
        """
2520.4.95 by Aaron Bentley
Add support for header/info records
148
        kwargs['storage_kind'] = 'header'
149
        self._add_record(None, kwargs, 'info', None, None)
150
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
151
    @staticmethod
2520.4.68 by Aaron Bentley
Change name separators to all-slash
152
    def encode_name(content_kind, revision_id, file_id=None):
2520.4.118 by Aaron Bentley
Add docs
153
        """Encode semantic ids as a container name"""
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
154
        if content_kind not in ('revision', 'file', 'inventory', 'signature',
155
                'info'):
156
            raise ValueError(content_kind)
2520.4.118 by Aaron Bentley
Add docs
157
        if content_kind == 'file':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
158
            if file_id is None:
159
                raise AssertionError()
2520.4.118 by Aaron Bentley
Add docs
160
        else:
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
161
            if file_id is not None:
162
                raise AssertionError()
2520.4.95 by Aaron Bentley
Add support for header/info records
163
        if content_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
164
            if revision_id is not None:
165
                raise AssertionError()
166
        elif revision_id is None:
167
            raise AssertionError()
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
168
        names = [n.replace('/', '//') for n in
169
                 (content_kind, revision_id, file_id) if n is not None]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
170
        return '/'.join(names)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
171
2520.4.56 by Aaron Bentley
Begin adding support for arbitrary metadata
172
    def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
173
        """Add a bundle record to the container.
174
175
        Most bundle records are recorded as header/body pairs, with the
176
        body being nameless.  Records with storage_kind 'header' have no
177
        body.
178
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
179
        name = self.encode_name(repo_kind, revision_id, file_id)
2520.4.95 by Aaron Bentley
Add support for header/info records
180
        encoded_metadata = bencode.bencode(metadata)
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
181
        self._container.add_bytes_record(encoded_metadata, [(name, )])
2520.4.95 by Aaron Bentley
Add support for header/info records
182
        if metadata['storage_kind'] != 'header':
183
            self._container.add_bytes_record(bytes, [])
2520.4.13 by Aaron Bentley
Use real container implementation
184
2520.4.7 by Aaron Bentley
Fix patch deserialization
185
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
186
class BundleReader(object):
2520.4.118 by Aaron Bentley
Add docs
187
    """Reader for bundle-format files.
188
189
    This serves roughly the same purpose as ContainerReader, but acts as a
190
    layer on top of it, providing metadata, a semantic name, and a record
191
    body
192
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
193
4543.2.14 by John Arbash Meinel
Clarify some comments, fix up a debugging change.
194
    def __init__(self, fileobj, stream_input=True):
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
195
        """Constructor
196
197
        :param fileobj: a file containing a bzip-encoded container
2520.4.148 by Aaron Bentley
Updates from review
198
        :param stream_input: If True, the BundleReader stream input rather than
199
            reading it all into memory at once.  Reading it into memory all at
200
            once is (currently) faster.
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
201
        """
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
202
        line = fileobj.readline()
203
        if line != '\n':
204
            fileobj.readline()
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
205
        self.patch_lines = []
2520.4.148 by Aaron Bentley
Updates from review
206
        if stream_input:
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
207
            source_file = iterablefile.IterableFile(self.iter_decode(fileobj))
208
        else:
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
209
            source_file = BytesIO(bz2.decompress(fileobj.read()))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
210
        self._container_file = source_file
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
211
212
    @staticmethod
213
    def iter_decode(fileobj):
2520.4.118 by Aaron Bentley
Add docs
214
        """Iterate through decoded fragments of the file"""
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
215
        decompressor = bz2.BZ2Decompressor()
216
        for line in fileobj:
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
217
            try:
218
                yield decompressor.decompress(line)
219
            except EOFError:
220
                return
2520.4.22 by Aaron Bentley
Create ContainerReader
221
222
    @staticmethod
223
    def decode_name(name):
2520.4.118 by Aaron Bentley
Add docs
224
        """Decode a name from its container form into a semantic form
225
226
        :retval: content_kind, revision_id, file_id
227
        """
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
228
        segments = re.split('(//?)', name)
229
        names = ['']
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
230
        for segment in segments:
231
            if segment == '//':
232
                names[-1] += '/'
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
233
            elif segment == '/':
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
234
                names.append('')
235
            else:
236
                names[-1] += segment
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
237
        content_kind = names[0]
2520.4.95 by Aaron Bentley
Add support for header/info records
238
        revision_id = None
239
        file_id = None
240
        if len(names) > 1:
241
            revision_id = names[1]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
242
        if len(names) > 2:
243
            file_id = names[2]
244
        return content_kind, revision_id, file_id
2520.4.22 by Aaron Bentley
Create ContainerReader
245
246
    def iter_records(self):
2520.4.118 by Aaron Bentley
Add docs
247
        """Iterate through bundle records
248
249
        :return: a generator of (bytes, metadata, content_kind, revision_id,
250
            file_id)
251
        """
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
252
        iterator = pack.iter_records_from_file(self._container_file)
253
        for names, bytes in iterator:
2520.4.131 by Aaron Bentley
Raise BadBundle for records with wrong number of names
254
            if len(names) != 1:
255
                raise errors.BadBundle('Record has %d names instead of 1'
256
                                       % len(names))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
257
            metadata = bencode.bdecode(bytes)
2520.4.95 by Aaron Bentley
Add support for header/info records
258
            if metadata['storage_kind'] == 'header':
259
                bytes = None
260
            else:
6634.2.1 by Martin
Apply 2to3 next fixer and make compatible
261
                _unused, bytes = next(iterator)
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
262
            yield (bytes, metadata) + self.decode_name(names[0][0])
2520.4.22 by Aaron Bentley
Create ContainerReader
263
264
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
265
class BundleSerializerV4(bundle_serializer.BundleSerializer):
2520.4.118 by Aaron Bentley
Add docs
266
    """Implement the high-level bundle interface"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
267
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
268
    def write(self, repository, revision_ids, forced_bases, fileobj):
2520.4.118 by Aaron Bentley
Add docs
269
        """Write a bundle to a file-like object
270
271
        For backwards-compatibility only
272
        """
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
273
        write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
274
                                                      forced_bases, fileobj)
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
275
        return write_op.do_write()
276
277
    def write_bundle(self, repository, target, base, fileobj):
2520.4.118 by Aaron Bentley
Add docs
278
        """Write a bundle to a file object
279
280
        :param repository: The repository to retrieve revision data from
281
        :param target: The head revision to include ancestors of
282
        :param base: The ancestor of the target to stop including acestors
283
            at.
284
        :param fileobj: The file-like object to write to
285
        """
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
286
        write_op =  BundleWriteOperation(base, target, repository, fileobj)
287
        return write_op.do_write()
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
288
289
    def read(self, file):
2520.4.118 by Aaron Bentley
Add docs
290
        """return a reader object for a given file"""
2520.4.72 by Aaron Bentley
Rename format to 4alpha
291
        bundle = BundleInfoV4(file, self)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
292
        return bundle
293
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
294
    @staticmethod
295
    def get_source_serializer(info):
2520.4.118 by Aaron Bentley
Add docs
296
        """Retrieve the serializer for a given info object"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
297
        return serializer.format_registry.get(info['serializer'])
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
298
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
299
300
class BundleWriteOperation(object):
2520.4.118 by Aaron Bentley
Add docs
301
    """Perform the operation of writing revisions to a bundle"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
302
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
303
    @classmethod
304
    def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
305
        """Create a BundleWriteOperation from old-style arguments"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
306
        base, target = cls.get_base_target(revision_ids, forced_bases,
307
                                           repository)
308
        return BundleWriteOperation(base, target, repository, fileobj,
309
                                    revision_ids)
310
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
311
    def __init__(self, base, target, repository, fileobj, revision_ids=None):
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
312
        self.base = base
313
        self.target = target
314
        self.repository = repository
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
315
        bundle = BundleWriter(fileobj)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
316
        self.bundle = bundle
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
317
        if revision_ids is not None:
318
            self.revision_ids = revision_ids
319
        else:
4154.1.1 by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice
320
            graph = repository.get_graph()
4154.1.3 by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again
321
            revision_ids = graph.find_unique_ancestors(target, [base])
322
            # Strip ghosts
323
            parents = graph.get_parent_map(revision_ids)
324
            self.revision_ids = [r for r in revision_ids if r in parents]
6619.3.12 by Jelmer Vernooij
Use 2to3 set_literal fixer.
325
        self.revision_keys = {(revid,) for revid in self.revision_ids}
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
326
327
    def do_write(self):
2520.4.118 by Aaron Bentley
Add docs
328
        """Write all data to the bundle"""
6143.1.3 by Jonathan Riddell
more plurals
329
        trace.note(ngettext('Bundling %d revision.', 'Bundling %d revisions.',
330
                            len(self.revision_ids)), len(self.revision_ids))
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
331
        self.repository.lock_read()
332
        try:
333
            self.bundle.begin()
334
            self.write_info()
335
            self.write_files()
336
            self.write_revisions()
337
            self.bundle.end()
338
        finally:
339
            self.repository.unlock()
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
340
        return self.revision_ids
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
341
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
342
    def write_info(self):
2520.4.118 by Aaron Bentley
Add docs
343
        """Write format info"""
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
344
        serializer_format = self.repository.get_serializer_format()
2520.4.99 by Aaron Bentley
Test conversion across models
345
        supports_rich_root = {True: 1, False: 0}[
346
            self.repository.supports_rich_root()]
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
347
        self.bundle.add_info_record(serializer=serializer_format,
2520.4.99 by Aaron Bentley
Test conversion across models
348
                                    supports_rich_root=supports_rich_root)
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
349
2520.4.51 by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it
350
    def write_files(self):
2520.4.118 by Aaron Bentley
Add docs
351
        """Write bundle records for all revisions of all files"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
352
        text_keys = []
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
353
        altered_fileids = self.repository.fileids_altered_by_revision_ids(
354
                self.revision_ids)
355
        for file_id, revision_ids in altered_fileids.iteritems():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
356
            for revision_id in revision_ids:
357
                text_keys.append((file_id, revision_id))
3350.6.10 by Martin Pool
VersionedFiles review cleanups
358
        self._add_mp_records_keys('file', self.repository.texts, text_keys)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
359
360
    def write_revisions(self):
2520.4.118 by Aaron Bentley
Add docs
361
        """Write bundle records for all revisions and signatures"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
362
        inv_vf = self.repository.inventories
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
363
        topological_order = [key[-1] for key in multiparent.topo_iter_keys(
364
                                inv_vf, self.revision_keys)]
365
        revision_order = topological_order
2520.4.75 by Aaron Bentley
Fix traceback on empty bundles.
366
        if self.target is not None and self.target in self.revision_ids:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
367
            # Make sure the target revision is always the last entry
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
368
            revision_order = list(topological_order)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
369
            revision_order.remove(self.target)
370
            revision_order.append(self.target)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
371
        if self.repository._serializer.support_altered_by_hack:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
372
            # Repositories that support_altered_by_hack means that
373
            # inventories.make_mpdiffs() contains all the data about the tree
374
            # shape. Formats without support_altered_by_hack require
375
            # chk_bytes/etc, so we use a different code path.
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
376
            self._add_mp_records_keys('inventory', inv_vf,
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
377
                                      [(revid,) for revid in topological_order])
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
378
        else:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
379
            # Inventories should always be added in pure-topological order, so
380
            # that we can apply the mpdiff for the child to the parent texts.
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
381
            self._add_inventory_mpdiffs_from_serializer(topological_order)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
382
        self._add_revision_texts(revision_order)
383
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
384
    def _add_inventory_mpdiffs_from_serializer(self, revision_order):
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
385
        """Generate mpdiffs by serializing inventories.
386
387
        The current repository only has part of the tree shape information in
388
        the 'inventories' vf. So we use serializer.write_inventory_to_string to
389
        get a 'full' representation of the tree shape, and then generate
390
        mpdiffs on that data stream. This stream can then be reconstructed on
391
        the other side.
392
        """
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
393
        inventory_key_order = [(r,) for r in revision_order]
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
394
        generator = _MPDiffInventoryGenerator(self.repository,
395
                                              inventory_key_order)
396
        for revision_id, parent_ids, sha1, diff in generator.iter_diffs():
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
397
            text = ''.join(diff.to_patch())
398
            self.bundle.add_multiparent_record(text, sha1, parent_ids,
399
                                               'inventory', revision_id, None)
400
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
401
    def _add_revision_texts(self, revision_order):
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
402
        parent_map = self.repository.get_parent_map(revision_order)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
403
        revision_to_str = self.repository._serializer.write_revision_to_string
404
        revisions = self.repository.get_revisions(revision_order)
405
        for revision in revisions:
406
            revision_id = revision.revision_id
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
407
            parents = parent_map.get(revision_id, None)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
408
            revision_text = revision_to_str(revision)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
409
            self.bundle.add_fulltext_record(revision_text, parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
410
                                       'revision', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
411
            try:
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
412
                self.bundle.add_fulltext_record(
413
                    self.repository.get_signature_text(
2520.4.123 by Aaron Bentley
Cleanup of bundle code
414
                    revision_id), parents, 'signature', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
415
            except errors.NoSuchRevision:
416
                pass
417
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
418
    @staticmethod
419
    def get_base_target(revision_ids, forced_bases, repository):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
420
        """Determine the base and target from old-style revision ids"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
421
        if len(revision_ids) == 0:
422
            return None, None
423
        target = revision_ids[0]
424
        base = forced_bases.get(target)
425
        if base is None:
426
            parents = repository.get_revision(target).parent_ids
427
            if len(parents) == 0:
428
                base = _mod_revision.NULL_REVISION
429
            else:
430
                base = parents[0]
431
        return base, target
432
3350.6.10 by Martin Pool
VersionedFiles review cleanups
433
    def _add_mp_records_keys(self, repo_kind, vf, keys):
2520.4.118 by Aaron Bentley
Add docs
434
        """Add multi-parent diff records to a bundle"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
435
        ordered_keys = list(multiparent.topo_iter_keys(vf, keys))
436
        mpdiffs = vf.make_mpdiffs(ordered_keys)
437
        sha1s = vf.get_sha1s(ordered_keys)
438
        parent_map = vf.get_parent_map(ordered_keys)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
439
        for mpdiff, item_key, in zip(mpdiffs, ordered_keys):
440
            sha1 = sha1s[item_key]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
441
            parents = [key[-1] for key in parent_map[item_key]]
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
442
            text = ''.join(mpdiff.to_patch())
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
443
            # Infer file id records as appropriate.
444
            if len(item_key) == 2:
445
                file_id = item_key[0]
446
            else:
447
                file_id = None
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
448
            self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
449
                                               item_key[-1], file_id)
2520.4.6 by Aaron Bentley
Get installation started
450
451
2520.4.72 by Aaron Bentley
Rename format to 4alpha
452
class BundleInfoV4(object):
2520.4.6 by Aaron Bentley
Get installation started
453
2520.4.118 by Aaron Bentley
Add docs
454
    """Provide (most of) the BundleInfo interface"""
2520.4.6 by Aaron Bentley
Get installation started
455
    def __init__(self, fileobj, serializer):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
456
        self._fileobj = fileobj
457
        self._serializer = serializer
458
        self.__real_revisions = None
459
        self.__revisions = None
460
461
    def install(self, repository):
462
        return self.install_revisions(repository)
463
2520.4.148 by Aaron Bentley
Updates from review
464
    def install_revisions(self, repository, stream_input=True):
465
        """Install this bundle's revisions into the specified repository
466
467
        :param target_repo: The repository to install into
468
        :param stream_input: If True, will stream input rather than reading it
469
            all into memory at once.  Reading it into memory all at once is
470
            (currently) faster.
471
        """
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
472
        repository.lock_write()
473
        try:
2520.4.148 by Aaron Bentley
Updates from review
474
            ri = RevisionInstaller(self.get_bundle_reader(stream_input),
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
475
                                   self._serializer, repository)
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
476
            return ri.install()
477
        finally:
478
            repository.unlock()
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
479
2520.4.109 by Aaron Bentley
start work on directive cherry-picking
480
    def get_merge_request(self, target_repo):
481
        """Provide data for performing a merge
482
483
        Returns suggested base, suggested target, and patch verification status
484
        """
485
        return None, self.target, 'inapplicable'
486
2520.4.148 by Aaron Bentley
Updates from review
487
    def get_bundle_reader(self, stream_input=True):
488
        """Return a new BundleReader for the associated bundle
489
490
        :param stream_input: If True, the BundleReader stream input rather than
491
            reading it all into memory at once.  Reading it into memory all at
492
            once is (currently) faster.
493
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
494
        self._fileobj.seek(0)
2520.4.148 by Aaron Bentley
Updates from review
495
        return BundleReader(self._fileobj, stream_input)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
496
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
497
    def _get_real_revisions(self):
498
        if self.__real_revisions is None:
499
            self.__real_revisions = []
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
500
            bundle_reader = self.get_bundle_reader()
2520.4.102 by Aaron Bentley
rename parents to metadata
501
            for bytes, metadata, repo_kind, revision_id, file_id in \
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
502
                bundle_reader.iter_records():
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
503
                if repo_kind == 'info':
504
                    serializer =\
2520.4.102 by Aaron Bentley
rename parents to metadata
505
                        self._serializer.get_source_serializer(metadata)
2520.4.22 by Aaron Bentley
Create ContainerReader
506
                if repo_kind == 'revision':
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
507
                    rev = serializer.read_revision_from_string(bytes)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
508
                    self.__real_revisions.append(rev)
509
        return self.__real_revisions
510
    real_revisions = property(_get_real_revisions)
511
512
    def _get_revisions(self):
513
        if self.__revisions is None:
514
            self.__revisions = []
515
            for revision in self.real_revisions:
2520.4.33 by Aaron Bentley
remove test dependencies on serialization minutia
516
                self.__revisions.append(
517
                    bundle_data.RevisionInfo.from_revision(revision))
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
518
        return self.__revisions
519
520
    revisions = property(_get_revisions)
521
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
522
    def _get_target(self):
523
        return self.revisions[-1].revision_id
524
525
    target = property(_get_target)
526
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
527
528
class RevisionInstaller(object):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
529
    """Installs revisions into a repository"""
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
530
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
531
    def __init__(self, container, serializer, repository):
532
        self._container = container
2520.4.6 by Aaron Bentley
Get installation started
533
        self._serializer = serializer
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
534
        self._repository = repository
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
535
        self._info = None
2520.4.99 by Aaron Bentley
Test conversion across models
536
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
537
    def install(self):
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
538
        """Perform the installation.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
539
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
540
        Must be called with the Repository locked.
541
        """
542
        self._repository.start_write_group()
543
        try:
2856.1.2 by Robert Collins
Review feedback.
544
            result = self._install_in_write_group()
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
545
        except:
546
            self._repository.abort_write_group()
547
            raise
548
        self._repository.commit_write_group()
549
        return result
550
2856.1.2 by Robert Collins
Review feedback.
551
    def _install_in_write_group(self):
2520.4.6 by Aaron Bentley
Get installation started
552
        current_file = None
553
        current_versionedfile = None
554
        pending_file_records = []
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
555
        inventory_vf = None
556
        pending_inventory_records = []
2520.4.8 by Aaron Bentley
Serialize inventory
557
        added_inv = set()
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
558
        target_revision = None
2520.4.58 by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind
559
        for bytes, metadata, repo_kind, revision_id, file_id in\
2520.4.22 by Aaron Bentley
Create ContainerReader
560
            self._container.iter_records():
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
561
            if repo_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
562
                if self._info is not None:
563
                    raise AssertionError()
2520.4.123 by Aaron Bentley
Cleanup of bundle code
564
                self._handle_info(metadata)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
565
            if (pending_file_records and
566
                (repo_kind, file_id) != ('file', current_file)):
567
                # Flush the data for a single file - prevents memory
568
                # spiking due to buffering all files in memory.
569
                self._install_mp_records_keys(self._repository.texts,
570
                    pending_file_records)
2520.4.8 by Aaron Bentley
Serialize inventory
571
                current_file = None
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
572
                del pending_file_records[:]
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
573
            if len(pending_inventory_records) > 0 and repo_kind != 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
574
                self._install_inventory_records(pending_inventory_records)
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
575
                pending_inventory_records = []
576
            if repo_kind == 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
577
                pending_inventory_records.append(((revision_id,), metadata, bytes))
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
578
            if repo_kind == 'revision':
579
                target_revision = revision_id
580
                self._install_revision(revision_id, metadata, bytes)
581
            if repo_kind == 'signature':
582
                self._install_signature(revision_id, metadata, bytes)
2520.4.22 by Aaron Bentley
Create ContainerReader
583
            if repo_kind == 'file':
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
584
                current_file = file_id
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
585
                pending_file_records.append(((file_id, revision_id), metadata, bytes))
586
        self._install_mp_records_keys(self._repository.texts, pending_file_records)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
587
        return target_revision
2520.4.6 by Aaron Bentley
Get installation started
588
2520.4.123 by Aaron Bentley
Cleanup of bundle code
589
    def _handle_info(self, info):
590
        """Extract data from an info record"""
591
        self._info = info
592
        self._source_serializer = self._serializer.get_source_serializer(info)
593
        if (info['supports_rich_root'] == 0 and
594
            self._repository.supports_rich_root()):
595
            self.update_root = True
596
        else:
597
            self.update_root = False
598
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
599
    def _install_mp_records(self, versionedfile, records):
2520.4.61 by Aaron Bentley
Do bulk insertion of records
600
        if len(records) == 0:
601
            return
602
        d_func = multiparent.MultiParent.from_patch
603
        vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
604
                      records if r not in versionedfile]
605
        versionedfile.add_mpdiffs(vf_records)
2520.4.8 by Aaron Bentley
Serialize inventory
606
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
607
    def _install_mp_records_keys(self, versionedfile, records):
608
        d_func = multiparent.MultiParent.from_patch
609
        vf_records = []
610
        for key, meta, text in records:
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
611
            # Adapt to tuple interface: A length two key is a file_id,
612
            # revision_id pair, a length 1 key is a
613
            # revision/signature/inventory. We need to do this because
614
            # the metadata extraction from the bundle has not yet been updated
615
            # to use the consistent tuple interface itself.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
616
            if len(key) == 2:
617
                prefix = key[:1]
618
            else:
619
                prefix = ()
620
            parents = [prefix + (parent,) for parent in meta['parents']]
621
            vf_records.append((key, parents, meta['sha1'], d_func(text)))
622
        versionedfile.add_mpdiffs(vf_records)
623
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
624
    def _get_parent_inventory_texts(self, inventory_text_cache,
625
                                    inventory_cache, parent_ids):
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
626
        cached_parent_texts = {}
627
        remaining_parent_ids = []
628
        for parent_id in parent_ids:
629
            p_text = inventory_text_cache.get(parent_id, None)
630
            if p_text is None:
631
                remaining_parent_ids.append(parent_id)
632
            else:
633
                cached_parent_texts[parent_id] = p_text
634
        ghosts = ()
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
635
        # TODO: Use inventory_cache to grab inventories we already have in
636
        #       memory
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
637
        if remaining_parent_ids:
638
            # first determine what keys are actually present in the local
639
            # inventories object (don't use revisions as they haven't been
640
            # installed yet.)
641
            parent_keys = [(r,) for r in remaining_parent_ids]
642
            present_parent_map = self._repository.inventories.get_parent_map(
643
                                        parent_keys)
644
            present_parent_ids = []
645
            ghosts = set()
646
            for p_id in remaining_parent_ids:
647
                if (p_id,) in present_parent_map:
648
                    present_parent_ids.append(p_id)
649
                else:
650
                    ghosts.add(p_id)
651
            to_string = self._source_serializer.write_inventory_to_string
652
            for parent_inv in self._repository.iter_inventories(
653
                                    present_parent_ids):
654
                p_text = to_string(parent_inv)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
655
                inventory_cache[parent_inv.revision_id] = parent_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
656
                cached_parent_texts[parent_inv.revision_id] = p_text
657
                inventory_text_cache[parent_inv.revision_id] = p_text
658
659
        parent_texts = [cached_parent_texts[parent_id]
660
                        for parent_id in parent_ids
661
                         if parent_id not in ghosts]
662
        return parent_texts
663
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
664
    def _install_inventory_records(self, records):
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
665
        if (self._info['serializer'] == self._repository._serializer.format_num
666
            and self._repository._serializer.support_altered_by_hack):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
667
            return self._install_mp_records_keys(self._repository.inventories,
668
                records)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
669
        # Use a 10MB text cache, since these are string xml inventories. Note
670
        # that 10MB is fairly small for large projects (a single inventory can
671
        # be >5MB). Another possibility is to cache 10-20 inventory texts
672
        # instead
673
        inventory_text_cache = lru_cache.LRUSizeCache(10*1024*1024)
4543.2.21 by John Arbash Meinel
A few more tiny tweaks to comments, etc.
674
        # Also cache the in-memory representation. This allows us to create
675
        # inventory deltas to apply rather than calling add_inventory from
676
        # scratch each time.
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
677
        inventory_cache = lru_cache.LRUCache(10)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
678
        pb = ui.ui_factory.nested_progress_bar()
679
        try:
680
            num_records = len(records)
681
            for idx, (key, metadata, bytes) in enumerate(records):
682
                pb.update('installing inventory', idx, num_records)
683
                revision_id = key[-1]
684
                parent_ids = metadata['parents']
685
                # Note: This assumes the local ghosts are identical to the
686
                #       ghosts in the source, as the Bundle serialization
687
                #       format doesn't record ghosts.
688
                p_texts = self._get_parent_inventory_texts(inventory_text_cache,
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
689
                                                           inventory_cache,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
690
                                                           parent_ids)
691
                # Why does to_lines() take strings as the source, it seems that
692
                # it would have to cast to a list of lines, which we get back
693
                # as lines and then cast back to a string.
694
                target_lines = multiparent.MultiParent.from_patch(bytes
695
                            ).to_lines(p_texts)
696
                inv_text = ''.join(target_lines)
697
                del target_lines
698
                sha1 = osutils.sha_string(inv_text)
699
                if sha1 != metadata['sha1']:
700
                    raise errors.BadBundle("Can't convert to target format")
701
                # Add this to the cache so we don't have to extract it again.
702
                inventory_text_cache[revision_id] = inv_text
703
                target_inv = self._source_serializer.read_inventory_from_string(
704
                    inv_text)
705
                self._handle_root(target_inv, parent_ids)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
706
                parent_inv = None
707
                if parent_ids:
708
                    parent_inv = inventory_cache.get(parent_ids[0], None)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
709
                try:
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
710
                    if parent_inv is None:
711
                        self._repository.add_inventory(revision_id, target_inv,
712
                                                       parent_ids)
713
                    else:
714
                        delta = target_inv._make_delta(parent_inv)
715
                        self._repository.add_inventory_by_delta(parent_ids[0],
716
                            delta, revision_id, parent_ids)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
717
                except errors.UnsupportedInventoryKind:
718
                    raise errors.IncompatibleRevision(repr(self._repository))
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
719
                inventory_cache[revision_id] = target_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
720
        finally:
721
            pb.finished()
2520.4.99 by Aaron Bentley
Test conversion across models
722
723
    def _handle_root(self, target_inv, parent_ids):
724
        revision_id = target_inv.revision_id
725
        if self.update_root:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
726
            text_key = (target_inv.root.file_id, revision_id)
727
            parent_keys = [(target_inv.root.file_id, parent) for
728
                parent in parent_ids]
729
            self._repository.texts.add_lines(text_key, parent_keys, [])
2520.4.99 by Aaron Bentley
Test conversion across models
730
        elif not self._repository.supports_rich_root():
731
            if target_inv.root.revision != revision_id:
732
                raise errors.IncompatibleRevision(repr(self._repository))
733
2520.4.59 by Aaron Bentley
Push metadata down the stack
734
    def _install_revision(self, revision_id, metadata, text):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
735
        if self._repository.has_revision(revision_id):
736
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
737
        revision = self._source_serializer.read_revision_from_string(text)
738
        self._repository.add_revision(revision.revision_id, revision)
2520.4.34 by Aaron Bentley
Add signature support
739
2520.4.59 by Aaron Bentley
Push metadata down the stack
740
    def _install_signature(self, revision_id, metadata, text):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
741
        transaction = self._repository.get_transaction()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
742
        if self._repository.has_signature_for_revision_id(revision_id):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
743
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
744
        self._repository.add_signature_text(revision_id, text)