/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
1
# Copyright (C) 2007-2010 Canonical Ltd
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
16
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
17
from cStringIO import StringIO
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
18
import bz2
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
19
import re
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
20
2520.4.13 by Aaron Bentley
Use real container implementation
21
from bzrlib import (
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
22
    diff,
2520.4.34 by Aaron Bentley
Add signature support
23
    errors,
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
24
    iterablefile,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
25
    lru_cache,
2520.4.13 by Aaron Bentley
Use real container implementation
26
    multiparent,
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
27
    osutils,
2520.4.13 by Aaron Bentley
Use real container implementation
28
    pack,
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
29
    revision as _mod_revision,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
30
    serializer,
2520.4.45 by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory
31
    trace,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
32
    ui,
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
33
    versionedfile as _mod_versionedfile,
2520.4.13 by Aaron Bentley
Use real container implementation
34
    )
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
35
from bzrlib.bundle import bundle_data, serializer as bundle_serializer
6138.3.5 by Jonathan Riddell
make the test suite pass
36
from bzrlib.i18n import gettext
2694.5.1 by Alexander Belchenko
pyrex bencode (without benchmarks)
37
from bzrlib import bencode
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
38
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
39
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
40
class _MPDiffInventoryGenerator(_mod_versionedfile._MPDiffGenerator):
41
    """Generate Inventory diffs serialized inventories."""
42
43
    def __init__(self, repo, inventory_keys):
44
        super(_MPDiffInventoryGenerator, self).__init__(repo.inventories,
45
            inventory_keys)
46
        self.repo = repo
47
        self.sha1s = {}
48
49
    def iter_diffs(self):
50
        """Compute the diffs one at a time."""
51
        # This is instead of compute_diffs() since we guarantee our ordering of
52
        # inventories, we don't have to do any buffering
53
        self._find_needed_keys()
54
        # We actually use a slightly different ordering. We grab all of the
55
        # parents first, and then grab the ordered requests.
56
        needed_ids = [k[-1] for k in self.present_parents]
57
        needed_ids.extend([k[-1] for k in self.ordered_keys])
58
        inv_to_str = self.repo._serializer.write_inventory_to_string
59
        for inv in self.repo.iter_inventories(needed_ids):
60
            revision_id = inv.revision_id
61
            key = (revision_id,)
62
            if key in self.present_parents:
63
                # Not a key we will transmit, which is a shame, since because
64
                # of that bundles don't work with stacked branches
65
                parent_ids = None
66
            else:
67
                parent_ids = [k[-1] for k in self.parent_map[key]]
68
            as_bytes = inv_to_str(inv)
69
            self._process_one_record(key, (as_bytes,))
70
            if parent_ids is None:
71
                continue
72
            diff = self.diffs.pop(key)
73
            sha1 = osutils.sha_string(as_bytes)
74
            yield revision_id, parent_ids, sha1, diff
75
76
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
77
class BundleWriter(object):
2520.4.118 by Aaron Bentley
Add docs
78
    """Writer for bundle-format files.
79
80
    This serves roughly the same purpose as ContainerReader, but acts as a
81
    layer on top of it.
82
2520.4.123 by Aaron Bentley
Cleanup of bundle code
83
    Provides ways of writing the specific record types supported this bundle
2520.4.118 by Aaron Bentley
Add docs
84
    format.
85
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
86
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
87
    def __init__(self, fileobj):
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
88
        self._container = pack.ContainerWriter(self._write_encoded)
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
89
        self._fileobj = fileobj
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
90
        self._compressor = bz2.BZ2Compressor()
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
91
2520.4.118 by Aaron Bentley
Add docs
92
    def _write_encoded(self, bytes):
93
        """Write bzip2-encoded bytes to the file"""
94
        self._fileobj.write(self._compressor.compress(bytes))
95
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
96
    def begin(self):
2520.4.118 by Aaron Bentley
Add docs
97
        """Start writing the bundle"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
98
        self._fileobj.write(bundle_serializer._get_bundle_header(
99
            bundle_serializer.v4_string))
2520.4.24 by Aaron Bentley
Move heading writing above container beginning
100
        self._fileobj.write('#\n')
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
101
        self._container.begin()
102
103
    def end(self):
2520.4.118 by Aaron Bentley
Add docs
104
        """Finish writing the bundle"""
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
105
        self._container.end()
2520.4.76 by Aaron Bentley
Move base64-encoding into merge directives
106
        self._fileobj.write(self._compressor.flush())
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
107
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
108
    def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
109
                               revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
110
        """Add a record for a multi-parent diff
111
112
        :mp_bytes: A multi-parent diff, as a bytestring
2520.4.123 by Aaron Bentley
Cleanup of bundle code
113
        :sha1: The sha1 hash of the fulltext
2520.4.118 by Aaron Bentley
Add docs
114
        :parents: a list of revision-ids of the parents
115
        :repo_kind: The kind of object in the repository.  May be 'file' or
116
            'inventory'
117
        :revision_id: The revision id of the mpdiff being added.
118
        :file_id: The file-id of the file, or None for inventories.
119
        """
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
120
        metadata = {'parents': parents,
121
                    'storage_kind': 'mpdiff',
122
                    'sha1': sha1}
123
        self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
124
2520.4.123 by Aaron Bentley
Cleanup of bundle code
125
    def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
2520.4.118 by Aaron Bentley
Add docs
126
        """Add a record for a fulltext
127
128
        :bytes: The fulltext, as a bytestring
129
        :parents: a list of revision-ids of the parents
130
        :repo_kind: The kind of object in the repository.  May be 'revision' or
131
            'signature'
132
        :revision_id: The revision id of the fulltext being added.
133
        """
134
        metadata = {'parents': parents,
2520.5.3 by Aaron Bentley
fix sha1 in bundle format 4
135
                    'storage_kind': 'mpdiff'}
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
136
        self._add_record(bytes, {'parents': parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
137
            'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
138
2520.4.95 by Aaron Bentley
Add support for header/info records
139
    def add_info_record(self, **kwargs):
2520.4.118 by Aaron Bentley
Add docs
140
        """Add an info record to the bundle
141
142
        Any parameters may be supplied, except 'self' and 'storage_kind'.
143
        Values must be lists, strings, integers, dicts, or a combination.
144
        """
2520.4.95 by Aaron Bentley
Add support for header/info records
145
        kwargs['storage_kind'] = 'header'
146
        self._add_record(None, kwargs, 'info', None, None)
147
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
148
    @staticmethod
2520.4.68 by Aaron Bentley
Change name separators to all-slash
149
    def encode_name(content_kind, revision_id, file_id=None):
2520.4.118 by Aaron Bentley
Add docs
150
        """Encode semantic ids as a container name"""
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
151
        if content_kind not in ('revision', 'file', 'inventory', 'signature',
152
                'info'):
153
            raise ValueError(content_kind)
2520.4.118 by Aaron Bentley
Add docs
154
        if content_kind == 'file':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
155
            if file_id is None:
156
                raise AssertionError()
2520.4.118 by Aaron Bentley
Add docs
157
        else:
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
158
            if file_id is not None:
159
                raise AssertionError()
2520.4.95 by Aaron Bentley
Add support for header/info records
160
        if content_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
161
            if revision_id is not None:
162
                raise AssertionError()
163
        elif revision_id is None:
164
            raise AssertionError()
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
165
        names = [n.replace('/', '//') for n in
166
                 (content_kind, revision_id, file_id) if n is not None]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
167
        return '/'.join(names)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
168
2520.4.56 by Aaron Bentley
Begin adding support for arbitrary metadata
169
    def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
170
        """Add a bundle record to the container.
171
172
        Most bundle records are recorded as header/body pairs, with the
173
        body being nameless.  Records with storage_kind 'header' have no
174
        body.
175
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
176
        name = self.encode_name(repo_kind, revision_id, file_id)
2520.4.95 by Aaron Bentley
Add support for header/info records
177
        encoded_metadata = bencode.bencode(metadata)
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
178
        self._container.add_bytes_record(encoded_metadata, [(name, )])
2520.4.95 by Aaron Bentley
Add support for header/info records
179
        if metadata['storage_kind'] != 'header':
180
            self._container.add_bytes_record(bytes, [])
2520.4.13 by Aaron Bentley
Use real container implementation
181
2520.4.7 by Aaron Bentley
Fix patch deserialization
182
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
183
class BundleReader(object):
2520.4.118 by Aaron Bentley
Add docs
184
    """Reader for bundle-format files.
185
186
    This serves roughly the same purpose as ContainerReader, but acts as a
187
    layer on top of it, providing metadata, a semantic name, and a record
188
    body
189
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
190
4543.2.14 by John Arbash Meinel
Clarify some comments, fix up a debugging change.
191
    def __init__(self, fileobj, stream_input=True):
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
192
        """Constructor
193
194
        :param fileobj: a file containing a bzip-encoded container
2520.4.148 by Aaron Bentley
Updates from review
195
        :param stream_input: If True, the BundleReader stream input rather than
196
            reading it all into memory at once.  Reading it into memory all at
197
            once is (currently) faster.
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
198
        """
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
199
        line = fileobj.readline()
200
        if line != '\n':
201
            fileobj.readline()
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
202
        self.patch_lines = []
2520.4.148 by Aaron Bentley
Updates from review
203
        if stream_input:
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
204
            source_file = iterablefile.IterableFile(self.iter_decode(fileobj))
205
        else:
206
            source_file = StringIO(bz2.decompress(fileobj.read()))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
207
        self._container_file = source_file
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
208
209
    @staticmethod
210
    def iter_decode(fileobj):
2520.4.118 by Aaron Bentley
Add docs
211
        """Iterate through decoded fragments of the file"""
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
212
        decompressor = bz2.BZ2Decompressor()
213
        for line in fileobj:
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
214
            try:
215
                yield decompressor.decompress(line)
216
            except EOFError:
217
                return
2520.4.22 by Aaron Bentley
Create ContainerReader
218
219
    @staticmethod
220
    def decode_name(name):
2520.4.118 by Aaron Bentley
Add docs
221
        """Decode a name from its container form into a semantic form
222
223
        :retval: content_kind, revision_id, file_id
224
        """
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
225
        segments = re.split('(//?)', name)
226
        names = ['']
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
227
        for segment in segments:
228
            if segment == '//':
229
                names[-1] += '/'
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
230
            elif segment == '/':
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
231
                names.append('')
232
            else:
233
                names[-1] += segment
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
234
        content_kind = names[0]
2520.4.95 by Aaron Bentley
Add support for header/info records
235
        revision_id = None
236
        file_id = None
237
        if len(names) > 1:
238
            revision_id = names[1]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
239
        if len(names) > 2:
240
            file_id = names[2]
241
        return content_kind, revision_id, file_id
2520.4.22 by Aaron Bentley
Create ContainerReader
242
243
    def iter_records(self):
2520.4.118 by Aaron Bentley
Add docs
244
        """Iterate through bundle records
245
246
        :return: a generator of (bytes, metadata, content_kind, revision_id,
247
            file_id)
248
        """
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
249
        iterator = pack.iter_records_from_file(self._container_file)
250
        for names, bytes in iterator:
2520.4.131 by Aaron Bentley
Raise BadBundle for records with wrong number of names
251
            if len(names) != 1:
252
                raise errors.BadBundle('Record has %d names instead of 1'
253
                                       % len(names))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
254
            metadata = bencode.bdecode(bytes)
2520.4.95 by Aaron Bentley
Add support for header/info records
255
            if metadata['storage_kind'] == 'header':
256
                bytes = None
257
            else:
258
                _unused, bytes = iterator.next()
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
259
            yield (bytes, metadata) + self.decode_name(names[0][0])
2520.4.22 by Aaron Bentley
Create ContainerReader
260
261
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
262
class BundleSerializerV4(bundle_serializer.BundleSerializer):
2520.4.118 by Aaron Bentley
Add docs
263
    """Implement the high-level bundle interface"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
264
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
265
    def write(self, repository, revision_ids, forced_bases, fileobj):
2520.4.118 by Aaron Bentley
Add docs
266
        """Write a bundle to a file-like object
267
268
        For backwards-compatibility only
269
        """
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
270
        write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
271
                                                      forced_bases, fileobj)
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
272
        return write_op.do_write()
273
274
    def write_bundle(self, repository, target, base, fileobj):
2520.4.118 by Aaron Bentley
Add docs
275
        """Write a bundle to a file object
276
277
        :param repository: The repository to retrieve revision data from
278
        :param target: The head revision to include ancestors of
279
        :param base: The ancestor of the target to stop including acestors
280
            at.
281
        :param fileobj: The file-like object to write to
282
        """
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
283
        write_op =  BundleWriteOperation(base, target, repository, fileobj)
284
        return write_op.do_write()
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
285
286
    def read(self, file):
2520.4.118 by Aaron Bentley
Add docs
287
        """return a reader object for a given file"""
2520.4.72 by Aaron Bentley
Rename format to 4alpha
288
        bundle = BundleInfoV4(file, self)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
289
        return bundle
290
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
291
    @staticmethod
292
    def get_source_serializer(info):
2520.4.118 by Aaron Bentley
Add docs
293
        """Retrieve the serializer for a given info object"""
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
294
        return serializer.format_registry.get(info['serializer'])
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
295
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
296
297
class BundleWriteOperation(object):
2520.4.118 by Aaron Bentley
Add docs
298
    """Perform the operation of writing revisions to a bundle"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
299
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
300
    @classmethod
301
    def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
302
        """Create a BundleWriteOperation from old-style arguments"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
303
        base, target = cls.get_base_target(revision_ids, forced_bases,
304
                                           repository)
305
        return BundleWriteOperation(base, target, repository, fileobj,
306
                                    revision_ids)
307
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
308
    def __init__(self, base, target, repository, fileobj, revision_ids=None):
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
309
        self.base = base
310
        self.target = target
311
        self.repository = repository
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
312
        bundle = BundleWriter(fileobj)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
313
        self.bundle = bundle
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
314
        if revision_ids is not None:
315
            self.revision_ids = revision_ids
316
        else:
4154.1.1 by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice
317
            graph = repository.get_graph()
4154.1.3 by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again
318
            revision_ids = graph.find_unique_ancestors(target, [base])
319
            # Strip ghosts
320
            parents = graph.get_parent_map(revision_ids)
321
            self.revision_ids = [r for r in revision_ids if r in parents]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
322
        self.revision_keys = set([(revid,) for revid in self.revision_ids])
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
323
324
    def do_write(self):
2520.4.118 by Aaron Bentley
Add docs
325
        """Write all data to the bundle"""
6138.3.1 by Jonathan Riddell
use gettext() in more files
326
        trace.note(gettext('Bundling %d revision(s).'), len(self.revision_ids))
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
327
        self.repository.lock_read()
328
        try:
329
            self.bundle.begin()
330
            self.write_info()
331
            self.write_files()
332
            self.write_revisions()
333
            self.bundle.end()
334
        finally:
335
            self.repository.unlock()
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
336
        return self.revision_ids
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
337
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
338
    def write_info(self):
2520.4.118 by Aaron Bentley
Add docs
339
        """Write format info"""
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
340
        serializer_format = self.repository.get_serializer_format()
2520.4.99 by Aaron Bentley
Test conversion across models
341
        supports_rich_root = {True: 1, False: 0}[
342
            self.repository.supports_rich_root()]
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
343
        self.bundle.add_info_record(serializer=serializer_format,
2520.4.99 by Aaron Bentley
Test conversion across models
344
                                    supports_rich_root=supports_rich_root)
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
345
2520.4.51 by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it
346
    def write_files(self):
2520.4.118 by Aaron Bentley
Add docs
347
        """Write bundle records for all revisions of all files"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
348
        text_keys = []
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
349
        altered_fileids = self.repository.fileids_altered_by_revision_ids(
350
                self.revision_ids)
351
        for file_id, revision_ids in altered_fileids.iteritems():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
352
            for revision_id in revision_ids:
353
                text_keys.append((file_id, revision_id))
3350.6.10 by Martin Pool
VersionedFiles review cleanups
354
        self._add_mp_records_keys('file', self.repository.texts, text_keys)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
355
356
    def write_revisions(self):
2520.4.118 by Aaron Bentley
Add docs
357
        """Write bundle records for all revisions and signatures"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
358
        inv_vf = self.repository.inventories
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
359
        topological_order = [key[-1] for key in multiparent.topo_iter_keys(
360
                                inv_vf, self.revision_keys)]
361
        revision_order = topological_order
2520.4.75 by Aaron Bentley
Fix traceback on empty bundles.
362
        if self.target is not None and self.target in self.revision_ids:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
363
            # Make sure the target revision is always the last entry
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
364
            revision_order = list(topological_order)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
365
            revision_order.remove(self.target)
366
            revision_order.append(self.target)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
367
        if self.repository._serializer.support_altered_by_hack:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
368
            # Repositories that support_altered_by_hack means that
369
            # inventories.make_mpdiffs() contains all the data about the tree
370
            # shape. Formats without support_altered_by_hack require
371
            # chk_bytes/etc, so we use a different code path.
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
372
            self._add_mp_records_keys('inventory', inv_vf,
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
373
                                      [(revid,) for revid in topological_order])
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
374
        else:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
375
            # Inventories should always be added in pure-topological order, so
376
            # that we can apply the mpdiff for the child to the parent texts.
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
377
            self._add_inventory_mpdiffs_from_serializer(topological_order)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
378
        self._add_revision_texts(revision_order)
379
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
380
    def _add_inventory_mpdiffs_from_serializer(self, revision_order):
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
381
        """Generate mpdiffs by serializing inventories.
382
383
        The current repository only has part of the tree shape information in
384
        the 'inventories' vf. So we use serializer.write_inventory_to_string to
385
        get a 'full' representation of the tree shape, and then generate
386
        mpdiffs on that data stream. This stream can then be reconstructed on
387
        the other side.
388
        """
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
389
        inventory_key_order = [(r,) for r in revision_order]
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
390
        generator = _MPDiffInventoryGenerator(self.repository,
391
                                              inventory_key_order)
392
        for revision_id, parent_ids, sha1, diff in generator.iter_diffs():
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
393
            text = ''.join(diff.to_patch())
394
            self.bundle.add_multiparent_record(text, sha1, parent_ids,
395
                                               'inventory', revision_id, None)
396
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
397
    def _add_revision_texts(self, revision_order):
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
398
        parent_map = self.repository.get_parent_map(revision_order)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
399
        revision_to_str = self.repository._serializer.write_revision_to_string
400
        revisions = self.repository.get_revisions(revision_order)
401
        for revision in revisions:
402
            revision_id = revision.revision_id
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
403
            parents = parent_map.get(revision_id, None)
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
404
            revision_text = revision_to_str(revision)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
405
            self.bundle.add_fulltext_record(revision_text, parents,
2520.4.123 by Aaron Bentley
Cleanup of bundle code
406
                                       'revision', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
407
            try:
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
408
                self.bundle.add_fulltext_record(
409
                    self.repository.get_signature_text(
2520.4.123 by Aaron Bentley
Cleanup of bundle code
410
                    revision_id), parents, 'signature', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
411
            except errors.NoSuchRevision:
412
                pass
413
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
414
    @staticmethod
415
    def get_base_target(revision_ids, forced_bases, repository):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
416
        """Determine the base and target from old-style revision ids"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
417
        if len(revision_ids) == 0:
418
            return None, None
419
        target = revision_ids[0]
420
        base = forced_bases.get(target)
421
        if base is None:
422
            parents = repository.get_revision(target).parent_ids
423
            if len(parents) == 0:
424
                base = _mod_revision.NULL_REVISION
425
            else:
426
                base = parents[0]
427
        return base, target
428
3350.6.10 by Martin Pool
VersionedFiles review cleanups
429
    def _add_mp_records_keys(self, repo_kind, vf, keys):
2520.4.118 by Aaron Bentley
Add docs
430
        """Add multi-parent diff records to a bundle"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
431
        ordered_keys = list(multiparent.topo_iter_keys(vf, keys))
432
        mpdiffs = vf.make_mpdiffs(ordered_keys)
433
        sha1s = vf.get_sha1s(ordered_keys)
434
        parent_map = vf.get_parent_map(ordered_keys)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
435
        for mpdiff, item_key, in zip(mpdiffs, ordered_keys):
436
            sha1 = sha1s[item_key]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
437
            parents = [key[-1] for key in parent_map[item_key]]
2520.4.41 by Aaron Bentley
Accelerate mpdiff generation
438
            text = ''.join(mpdiff.to_patch())
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
439
            # Infer file id records as appropriate.
440
            if len(item_key) == 2:
441
                file_id = item_key[0]
442
            else:
443
                file_id = None
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
444
            self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
445
                                               item_key[-1], file_id)
2520.4.6 by Aaron Bentley
Get installation started
446
447
2520.4.72 by Aaron Bentley
Rename format to 4alpha
448
class BundleInfoV4(object):
2520.4.6 by Aaron Bentley
Get installation started
449
2520.4.118 by Aaron Bentley
Add docs
450
    """Provide (most of) the BundleInfo interface"""
2520.4.6 by Aaron Bentley
Get installation started
451
    def __init__(self, fileobj, serializer):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
452
        self._fileobj = fileobj
453
        self._serializer = serializer
454
        self.__real_revisions = None
455
        self.__revisions = None
456
457
    def install(self, repository):
458
        return self.install_revisions(repository)
459
2520.4.148 by Aaron Bentley
Updates from review
460
    def install_revisions(self, repository, stream_input=True):
461
        """Install this bundle's revisions into the specified repository
462
463
        :param target_repo: The repository to install into
464
        :param stream_input: If True, will stream input rather than reading it
465
            all into memory at once.  Reading it into memory all at once is
466
            (currently) faster.
467
        """
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
468
        repository.lock_write()
469
        try:
2520.4.148 by Aaron Bentley
Updates from review
470
            ri = RevisionInstaller(self.get_bundle_reader(stream_input),
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
471
                                   self._serializer, repository)
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
472
            return ri.install()
473
        finally:
474
            repository.unlock()
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
475
2520.4.109 by Aaron Bentley
start work on directive cherry-picking
476
    def get_merge_request(self, target_repo):
477
        """Provide data for performing a merge
478
479
        Returns suggested base, suggested target, and patch verification status
480
        """
481
        return None, self.target, 'inapplicable'
482
2520.4.148 by Aaron Bentley
Updates from review
483
    def get_bundle_reader(self, stream_input=True):
484
        """Return a new BundleReader for the associated bundle
485
486
        :param stream_input: If True, the BundleReader stream input rather than
487
            reading it all into memory at once.  Reading it into memory all at
488
            once is (currently) faster.
489
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
490
        self._fileobj.seek(0)
2520.4.148 by Aaron Bentley
Updates from review
491
        return BundleReader(self._fileobj, stream_input)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
492
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
493
    def _get_real_revisions(self):
494
        if self.__real_revisions is None:
495
            self.__real_revisions = []
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
496
            bundle_reader = self.get_bundle_reader()
2520.4.102 by Aaron Bentley
rename parents to metadata
497
            for bytes, metadata, repo_kind, revision_id, file_id in \
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
498
                bundle_reader.iter_records():
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
499
                if repo_kind == 'info':
500
                    serializer =\
2520.4.102 by Aaron Bentley
rename parents to metadata
501
                        self._serializer.get_source_serializer(metadata)
2520.4.22 by Aaron Bentley
Create ContainerReader
502
                if repo_kind == 'revision':
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
503
                    rev = serializer.read_revision_from_string(bytes)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
504
                    self.__real_revisions.append(rev)
505
        return self.__real_revisions
506
    real_revisions = property(_get_real_revisions)
507
508
    def _get_revisions(self):
509
        if self.__revisions is None:
510
            self.__revisions = []
511
            for revision in self.real_revisions:
2520.4.33 by Aaron Bentley
remove test dependencies on serialization minutia
512
                self.__revisions.append(
513
                    bundle_data.RevisionInfo.from_revision(revision))
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
514
        return self.__revisions
515
516
    revisions = property(_get_revisions)
517
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
518
    def _get_target(self):
519
        return self.revisions[-1].revision_id
520
521
    target = property(_get_target)
522
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
523
524
class RevisionInstaller(object):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
525
    """Installs revisions into a repository"""
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
526
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
527
    def __init__(self, container, serializer, repository):
528
        self._container = container
2520.4.6 by Aaron Bentley
Get installation started
529
        self._serializer = serializer
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
530
        self._repository = repository
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
531
        self._info = None
2520.4.99 by Aaron Bentley
Test conversion across models
532
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
533
    def install(self):
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
534
        """Perform the installation.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
535
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
536
        Must be called with the Repository locked.
537
        """
538
        self._repository.start_write_group()
539
        try:
2856.1.2 by Robert Collins
Review feedback.
540
            result = self._install_in_write_group()
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
541
        except:
542
            self._repository.abort_write_group()
543
            raise
544
        self._repository.commit_write_group()
545
        return result
546
2856.1.2 by Robert Collins
Review feedback.
547
    def _install_in_write_group(self):
2520.4.6 by Aaron Bentley
Get installation started
548
        current_file = None
549
        current_versionedfile = None
550
        pending_file_records = []
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
551
        inventory_vf = None
552
        pending_inventory_records = []
2520.4.8 by Aaron Bentley
Serialize inventory
553
        added_inv = set()
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
554
        target_revision = None
2520.4.58 by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind
555
        for bytes, metadata, repo_kind, revision_id, file_id in\
2520.4.22 by Aaron Bentley
Create ContainerReader
556
            self._container.iter_records():
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
557
            if repo_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
558
                if self._info is not None:
559
                    raise AssertionError()
2520.4.123 by Aaron Bentley
Cleanup of bundle code
560
                self._handle_info(metadata)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
561
            if (pending_file_records and
562
                (repo_kind, file_id) != ('file', current_file)):
563
                # Flush the data for a single file - prevents memory
564
                # spiking due to buffering all files in memory.
565
                self._install_mp_records_keys(self._repository.texts,
566
                    pending_file_records)
2520.4.8 by Aaron Bentley
Serialize inventory
567
                current_file = None
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
568
                del pending_file_records[:]
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
569
            if len(pending_inventory_records) > 0 and repo_kind != 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
570
                self._install_inventory_records(pending_inventory_records)
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
571
                pending_inventory_records = []
572
            if repo_kind == 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
573
                pending_inventory_records.append(((revision_id,), metadata, bytes))
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
574
            if repo_kind == 'revision':
575
                target_revision = revision_id
576
                self._install_revision(revision_id, metadata, bytes)
577
            if repo_kind == 'signature':
578
                self._install_signature(revision_id, metadata, bytes)
2520.4.22 by Aaron Bentley
Create ContainerReader
579
            if repo_kind == 'file':
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
580
                current_file = file_id
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
581
                pending_file_records.append(((file_id, revision_id), metadata, bytes))
582
        self._install_mp_records_keys(self._repository.texts, pending_file_records)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
583
        return target_revision
2520.4.6 by Aaron Bentley
Get installation started
584
2520.4.123 by Aaron Bentley
Cleanup of bundle code
585
    def _handle_info(self, info):
586
        """Extract data from an info record"""
587
        self._info = info
588
        self._source_serializer = self._serializer.get_source_serializer(info)
589
        if (info['supports_rich_root'] == 0 and
590
            self._repository.supports_rich_root()):
591
            self.update_root = True
592
        else:
593
            self.update_root = False
594
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
595
    def _install_mp_records(self, versionedfile, records):
2520.4.61 by Aaron Bentley
Do bulk insertion of records
596
        if len(records) == 0:
597
            return
598
        d_func = multiparent.MultiParent.from_patch
599
        vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
600
                      records if r not in versionedfile]
601
        versionedfile.add_mpdiffs(vf_records)
2520.4.8 by Aaron Bentley
Serialize inventory
602
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
603
    def _install_mp_records_keys(self, versionedfile, records):
604
        d_func = multiparent.MultiParent.from_patch
605
        vf_records = []
606
        for key, meta, text in records:
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
607
            # Adapt to tuple interface: A length two key is a file_id,
608
            # revision_id pair, a length 1 key is a
609
            # revision/signature/inventory. We need to do this because
610
            # the metadata extraction from the bundle has not yet been updated
611
            # to use the consistent tuple interface itself.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
612
            if len(key) == 2:
613
                prefix = key[:1]
614
            else:
615
                prefix = ()
616
            parents = [prefix + (parent,) for parent in meta['parents']]
617
            vf_records.append((key, parents, meta['sha1'], d_func(text)))
618
        versionedfile.add_mpdiffs(vf_records)
619
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
620
    def _get_parent_inventory_texts(self, inventory_text_cache,
621
                                    inventory_cache, parent_ids):
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
622
        cached_parent_texts = {}
623
        remaining_parent_ids = []
624
        for parent_id in parent_ids:
625
            p_text = inventory_text_cache.get(parent_id, None)
626
            if p_text is None:
627
                remaining_parent_ids.append(parent_id)
628
            else:
629
                cached_parent_texts[parent_id] = p_text
630
        ghosts = ()
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
631
        # TODO: Use inventory_cache to grab inventories we already have in
632
        #       memory
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
633
        if remaining_parent_ids:
634
            # first determine what keys are actually present in the local
635
            # inventories object (don't use revisions as they haven't been
636
            # installed yet.)
637
            parent_keys = [(r,) for r in remaining_parent_ids]
638
            present_parent_map = self._repository.inventories.get_parent_map(
639
                                        parent_keys)
640
            present_parent_ids = []
641
            ghosts = set()
642
            for p_id in remaining_parent_ids:
643
                if (p_id,) in present_parent_map:
644
                    present_parent_ids.append(p_id)
645
                else:
646
                    ghosts.add(p_id)
647
            to_string = self._source_serializer.write_inventory_to_string
648
            for parent_inv in self._repository.iter_inventories(
649
                                    present_parent_ids):
650
                p_text = to_string(parent_inv)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
651
                inventory_cache[parent_inv.revision_id] = parent_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
652
                cached_parent_texts[parent_inv.revision_id] = p_text
653
                inventory_text_cache[parent_inv.revision_id] = p_text
654
655
        parent_texts = [cached_parent_texts[parent_id]
656
                        for parent_id in parent_ids
657
                         if parent_id not in ghosts]
658
        return parent_texts
659
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
660
    def _install_inventory_records(self, records):
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
661
        if (self._info['serializer'] == self._repository._serializer.format_num
662
            and self._repository._serializer.support_altered_by_hack):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
663
            return self._install_mp_records_keys(self._repository.inventories,
664
                records)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
665
        # Use a 10MB text cache, since these are string xml inventories. Note
666
        # that 10MB is fairly small for large projects (a single inventory can
667
        # be >5MB). Another possibility is to cache 10-20 inventory texts
668
        # instead
669
        inventory_text_cache = lru_cache.LRUSizeCache(10*1024*1024)
4543.2.21 by John Arbash Meinel
A few more tiny tweaks to comments, etc.
670
        # Also cache the in-memory representation. This allows us to create
671
        # inventory deltas to apply rather than calling add_inventory from
672
        # scratch each time.
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
673
        inventory_cache = lru_cache.LRUCache(10)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
674
        pb = ui.ui_factory.nested_progress_bar()
675
        try:
676
            num_records = len(records)
677
            for idx, (key, metadata, bytes) in enumerate(records):
678
                pb.update('installing inventory', idx, num_records)
679
                revision_id = key[-1]
680
                parent_ids = metadata['parents']
681
                # Note: This assumes the local ghosts are identical to the
682
                #       ghosts in the source, as the Bundle serialization
683
                #       format doesn't record ghosts.
684
                p_texts = self._get_parent_inventory_texts(inventory_text_cache,
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
685
                                                           inventory_cache,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
686
                                                           parent_ids)
687
                # Why does to_lines() take strings as the source, it seems that
688
                # it would have to cast to a list of lines, which we get back
689
                # as lines and then cast back to a string.
690
                target_lines = multiparent.MultiParent.from_patch(bytes
691
                            ).to_lines(p_texts)
692
                inv_text = ''.join(target_lines)
693
                del target_lines
694
                sha1 = osutils.sha_string(inv_text)
695
                if sha1 != metadata['sha1']:
696
                    raise errors.BadBundle("Can't convert to target format")
697
                # Add this to the cache so we don't have to extract it again.
698
                inventory_text_cache[revision_id] = inv_text
699
                target_inv = self._source_serializer.read_inventory_from_string(
700
                    inv_text)
701
                self._handle_root(target_inv, parent_ids)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
702
                parent_inv = None
703
                if parent_ids:
704
                    parent_inv = inventory_cache.get(parent_ids[0], None)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
705
                try:
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
706
                    if parent_inv is None:
707
                        self._repository.add_inventory(revision_id, target_inv,
708
                                                       parent_ids)
709
                    else:
710
                        delta = target_inv._make_delta(parent_inv)
711
                        self._repository.add_inventory_by_delta(parent_ids[0],
712
                            delta, revision_id, parent_ids)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
713
                except errors.UnsupportedInventoryKind:
714
                    raise errors.IncompatibleRevision(repr(self._repository))
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
715
                inventory_cache[revision_id] = target_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
716
        finally:
717
            pb.finished()
2520.4.99 by Aaron Bentley
Test conversion across models
718
719
    def _handle_root(self, target_inv, parent_ids):
720
        revision_id = target_inv.revision_id
721
        if self.update_root:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
722
            text_key = (target_inv.root.file_id, revision_id)
723
            parent_keys = [(target_inv.root.file_id, parent) for
724
                parent in parent_ids]
725
            self._repository.texts.add_lines(text_key, parent_keys, [])
2520.4.99 by Aaron Bentley
Test conversion across models
726
        elif not self._repository.supports_rich_root():
727
            if target_inv.root.revision != revision_id:
728
                raise errors.IncompatibleRevision(repr(self._repository))
729
2520.4.59 by Aaron Bentley
Push metadata down the stack
730
    def _install_revision(self, revision_id, metadata, text):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
731
        if self._repository.has_revision(revision_id):
732
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
733
        revision = self._source_serializer.read_revision_from_string(text)
734
        self._repository.add_revision(revision.revision_id, revision)
2520.4.34 by Aaron Bentley
Add signature support
735
2520.4.59 by Aaron Bentley
Push metadata down the stack
736
    def _install_signature(self, revision_id, metadata, text):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
737
        transaction = self._repository.get_transaction()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
738
        if self._repository.has_signature_for_revision_id(revision_id):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
739
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
740
        self._repository.add_signature_text(revision_id, text)