/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
1
# Copyright (C) 2007-2010 Canonical Ltd
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2520.4.85 by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!)
16
6379.6.3 by Jelmer Vernooij
Use absolute_import.
17
from __future__ import absolute_import
18
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
19
import bz2
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
20
import re
2520.4.20 by Aaron Bentley
Compress and base64-encode bundle contents
21
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
22
from ... import (
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
23
    bencode,
2520.4.34 by Aaron Bentley
Add signature support
24
    errors,
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
25
    iterablefile,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
26
    lru_cache,
2520.4.13 by Aaron Bentley
Use real container implementation
27
    multiparent,
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
28
    osutils,
7143.21.1 by Jelmer Vernooij
Add WriteGroup contextmanager.
29
    repository as _mod_repository,
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
30
    revision as _mod_revision,
2520.4.45 by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory
31
    trace,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
32
    ui,
6670.4.1 by Jelmer Vernooij
Update imports.
33
    )
34
from ...bzr import (
35
    pack,
6670.4.10 by Jelmer Vernooij
Move serializer to bzr.
36
    serializer,
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
37
    versionedfile as _mod_versionedfile,
2520.4.13 by Aaron Bentley
Use real container implementation
38
    )
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
39
from ...bundle import bundle_data, serializer as bundle_serializer
40
from ...i18n import ngettext
41
from ...sixish import (
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
42
    BytesIO,
6656.1.1 by Martin
Apply 2to3 dict fixer and clean up resulting mess using view helpers
43
    viewitems,
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
44
    )
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
45
2520.4.4 by Aaron Bentley
Get basis support for a new bundle format in place
46
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
47
class _MPDiffInventoryGenerator(_mod_versionedfile._MPDiffGenerator):
48
    """Generate Inventory diffs serialized inventories."""
49
50
    def __init__(self, repo, inventory_keys):
51
        super(_MPDiffInventoryGenerator, self).__init__(repo.inventories,
7143.15.2 by Jelmer Vernooij
Run autopep8.
52
                                                        inventory_keys)
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
53
        self.repo = repo
54
        self.sha1s = {}
55
56
    def iter_diffs(self):
57
        """Compute the diffs one at a time."""
58
        # This is instead of compute_diffs() since we guarantee our ordering of
59
        # inventories, we don't have to do any buffering
60
        self._find_needed_keys()
61
        # We actually use a slightly different ordering. We grab all of the
62
        # parents first, and then grab the ordered requests.
63
        needed_ids = [k[-1] for k in self.present_parents]
64
        needed_ids.extend([k[-1] for k in self.ordered_keys])
7027.3.3 by Jelmer Vernooij
Add some more bees; support writing both bytes and unicode strings in build_tree_contents.
65
        inv_to_bytes = self.repo._serializer.write_inventory_to_string
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
66
        for inv in self.repo.iter_inventories(needed_ids):
67
            revision_id = inv.revision_id
68
            key = (revision_id,)
69
            if key in self.present_parents:
70
                # Not a key we will transmit, which is a shame, since because
71
                # of that bundles don't work with stacked branches
72
                parent_ids = None
73
            else:
74
                parent_ids = [k[-1] for k in self.parent_map[key]]
7027.3.3 by Jelmer Vernooij
Add some more bees; support writing both bytes and unicode strings in build_tree_contents.
75
            as_bytes = inv_to_bytes(inv)
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
76
            self._process_one_record(key, (as_bytes,))
77
            if parent_ids is None:
78
                continue
79
            diff = self.diffs.pop(key)
80
            sha1 = osutils.sha_string(as_bytes)
81
            yield revision_id, parent_ids, sha1, diff
82
83
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
84
class BundleWriter(object):
2520.4.118 by Aaron Bentley
Add docs
85
    """Writer for bundle-format files.
86
87
    This serves roughly the same purpose as ContainerReader, but acts as a
88
    layer on top of it.
89
2520.4.123 by Aaron Bentley
Cleanup of bundle code
90
    Provides ways of writing the specific record types supported this bundle
2520.4.118 by Aaron Bentley
Add docs
91
    format.
92
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
93
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
94
    def __init__(self, fileobj):
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
95
        self._container = pack.ContainerWriter(self._write_encoded)
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
96
        self._fileobj = fileobj
2520.4.27 by Aaron Bentley
Use less memory when writing bzip-encoded files
97
        self._compressor = bz2.BZ2Compressor()
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
98
2520.4.118 by Aaron Bentley
Add docs
99
    def _write_encoded(self, bytes):
100
        """Write bzip2-encoded bytes to the file"""
101
        self._fileobj.write(self._compressor.compress(bytes))
102
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
103
    def begin(self):
2520.4.118 by Aaron Bentley
Add docs
104
        """Start writing the bundle"""
6989.1.1 by Jelmer Vernooij
Use format registry for bundles.
105
        self._fileobj.write(bundle_serializer._get_bundle_header('4'))
6973.6.1 by Jelmer Vernooij
More bees.
106
        self._fileobj.write(b'#\n')
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
107
        self._container.begin()
108
109
    def end(self):
2520.4.118 by Aaron Bentley
Add docs
110
        """Finish writing the bundle"""
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
111
        self._container.end()
2520.4.76 by Aaron Bentley
Move base64-encoding into merge directives
112
        self._fileobj.write(self._compressor.flush())
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
113
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
114
    def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
115
                               revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
116
        """Add a record for a multi-parent diff
117
118
        :mp_bytes: A multi-parent diff, as a bytestring
2520.4.123 by Aaron Bentley
Cleanup of bundle code
119
        :sha1: The sha1 hash of the fulltext
2520.4.118 by Aaron Bentley
Add docs
120
        :parents: a list of revision-ids of the parents
121
        :repo_kind: The kind of object in the repository.  May be 'file' or
122
            'inventory'
123
        :revision_id: The revision id of the mpdiff being added.
124
        :file_id: The file-id of the file, or None for inventories.
125
        """
6973.7.8 by Jelmer Vernooij
Fix more tests.
126
        metadata = {b'parents': parents,
127
                    b'storage_kind': b'mpdiff',
128
                    b'sha1': sha1}
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
129
        self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
130
2520.4.123 by Aaron Bentley
Cleanup of bundle code
131
    def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
2520.4.118 by Aaron Bentley
Add docs
132
        """Add a record for a fulltext
133
134
        :bytes: The fulltext, as a bytestring
135
        :parents: a list of revision-ids of the parents
136
        :repo_kind: The kind of object in the repository.  May be 'revision' or
137
            'signature'
138
        :revision_id: The revision id of the fulltext being added.
139
        """
6973.7.8 by Jelmer Vernooij
Fix more tests.
140
        metadata = {b'parents': parents,
141
                    b'storage_kind': b'mpdiff'}
142
        self._add_record(bytes, {b'parents': parents,
7143.15.2 by Jelmer Vernooij
Run autopep8.
143
                                 b'storage_kind': b'fulltext'}, repo_kind, revision_id, None)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
144
6973.7.8 by Jelmer Vernooij
Fix more tests.
145
    def add_info_record(self, kwargs):
2520.4.118 by Aaron Bentley
Add docs
146
        """Add an info record to the bundle
147
148
        Any parameters may be supplied, except 'self' and 'storage_kind'.
149
        Values must be lists, strings, integers, dicts, or a combination.
150
        """
6973.7.8 by Jelmer Vernooij
Fix more tests.
151
        kwargs[b'storage_kind'] = b'header'
2520.4.95 by Aaron Bentley
Add support for header/info records
152
        self._add_record(None, kwargs, 'info', None, None)
153
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
154
    @staticmethod
2520.4.68 by Aaron Bentley
Change name separators to all-slash
155
    def encode_name(content_kind, revision_id, file_id=None):
2520.4.118 by Aaron Bentley
Add docs
156
        """Encode semantic ids as a container name"""
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
157
        if content_kind not in ('revision', 'file', 'inventory', 'signature',
7143.15.2 by Jelmer Vernooij
Run autopep8.
158
                                'info'):
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
159
            raise ValueError(content_kind)
2520.4.118 by Aaron Bentley
Add docs
160
        if content_kind == 'file':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
161
            if file_id is None:
162
                raise AssertionError()
2520.4.118 by Aaron Bentley
Add docs
163
        else:
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
164
            if file_id is not None:
165
                raise AssertionError()
2520.4.95 by Aaron Bentley
Add support for header/info records
166
        if content_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
167
            if revision_id is not None:
168
                raise AssertionError()
169
        elif revision_id is None:
170
            raise AssertionError()
6973.7.8 by Jelmer Vernooij
Fix more tests.
171
        names = [n.replace(b'/', b'//') for n in
172
                 (content_kind.encode('ascii'), revision_id, file_id) if n is not None]
173
        return b'/'.join(names)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
174
2520.4.56 by Aaron Bentley
Begin adding support for arbitrary metadata
175
    def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
2520.4.118 by Aaron Bentley
Add docs
176
        """Add a bundle record to the container.
177
178
        Most bundle records are recorded as header/body pairs, with the
179
        body being nameless.  Records with storage_kind 'header' have no
180
        body.
181
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
182
        name = self.encode_name(repo_kind, revision_id, file_id)
2520.4.95 by Aaron Bentley
Add support for header/info records
183
        encoded_metadata = bencode.bencode(metadata)
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
184
        self._container.add_bytes_record(encoded_metadata, [(name, )])
6973.7.8 by Jelmer Vernooij
Fix more tests.
185
        if metadata[b'storage_kind'] != b'header':
2520.4.95 by Aaron Bentley
Add support for header/info records
186
            self._container.add_bytes_record(bytes, [])
2520.4.13 by Aaron Bentley
Use real container implementation
187
2520.4.7 by Aaron Bentley
Fix patch deserialization
188
2520.4.25 by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader
189
class BundleReader(object):
2520.4.118 by Aaron Bentley
Add docs
190
    """Reader for bundle-format files.
191
192
    This serves roughly the same purpose as ContainerReader, but acts as a
193
    layer on top of it, providing metadata, a semantic name, and a record
194
    body
195
    """
2520.4.123 by Aaron Bentley
Cleanup of bundle code
196
4543.2.14 by John Arbash Meinel
Clarify some comments, fix up a debugging change.
197
    def __init__(self, fileobj, stream_input=True):
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
198
        """Constructor
199
200
        :param fileobj: a file containing a bzip-encoded container
2520.4.148 by Aaron Bentley
Updates from review
201
        :param stream_input: If True, the BundleReader stream input rather than
202
            reading it all into memory at once.  Reading it into memory all at
203
            once is (currently) faster.
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
204
        """
2520.4.23 by Aaron Bentley
Move responsability for encoding into container objects
205
        line = fileobj.readline()
206
        if line != '\n':
207
            fileobj.readline()
2520.4.40 by Aaron Bentley
Add human-readable diff to bundles
208
        self.patch_lines = []
2520.4.148 by Aaron Bentley
Updates from review
209
        if stream_input:
2520.4.145 by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives
210
            source_file = iterablefile.IterableFile(self.iter_decode(fileobj))
211
        else:
6621.22.2 by Martin
Use BytesIO or StringIO from bzrlib.sixish
212
            source_file = BytesIO(bz2.decompress(fileobj.read()))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
213
        self._container_file = source_file
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
214
215
    @staticmethod
216
    def iter_decode(fileobj):
2520.4.118 by Aaron Bentley
Add docs
217
        """Iterate through decoded fragments of the file"""
2520.4.26 by Aaron Bentley
Make decompression reasonably memory-efficient
218
        decompressor = bz2.BZ2Decompressor()
219
        for line in fileobj:
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
220
            try:
221
                yield decompressor.decompress(line)
222
            except EOFError:
223
                return
2520.4.22 by Aaron Bentley
Create ContainerReader
224
225
    @staticmethod
226
    def decode_name(name):
2520.4.118 by Aaron Bentley
Add docs
227
        """Decode a name from its container form into a semantic form
228
229
        :retval: content_kind, revision_id, file_id
230
        """
6973.7.10 by Jelmer Vernooij
More fixes.
231
        segments = re.split(b'(//?)', name)
232
        names = [b'']
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
233
        for segment in segments:
6973.7.10 by Jelmer Vernooij
More fixes.
234
            if segment == b'//':
235
                names[-1] += b'/'
236
            elif segment == b'/':
237
                names.append(b'')
2520.4.127 by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes
238
            else:
239
                names[-1] += segment
2520.4.130 by Aaron Bentley
Finish tweaking decode_name
240
        content_kind = names[0]
2520.4.95 by Aaron Bentley
Add support for header/info records
241
        revision_id = None
242
        file_id = None
243
        if len(names) > 1:
244
            revision_id = names[1]
2520.4.68 by Aaron Bentley
Change name separators to all-slash
245
        if len(names) > 2:
246
            file_id = names[2]
6973.7.10 by Jelmer Vernooij
More fixes.
247
        return content_kind.decode('ascii'), revision_id, file_id
2520.4.22 by Aaron Bentley
Create ContainerReader
248
249
    def iter_records(self):
2520.4.118 by Aaron Bentley
Add docs
250
        """Iterate through bundle records
251
252
        :return: a generator of (bytes, metadata, content_kind, revision_id,
253
            file_id)
254
        """
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
255
        iterator = pack.iter_records_from_file(self._container_file)
256
        for names, bytes in iterator:
2520.4.131 by Aaron Bentley
Raise BadBundle for records with wrong number of names
257
            if len(names) != 1:
258
                raise errors.BadBundle('Record has %d names instead of 1'
259
                                       % len(names))
2916.2.18 by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader.
260
            metadata = bencode.bdecode(bytes)
6973.7.10 by Jelmer Vernooij
More fixes.
261
            if metadata[b'storage_kind'] == b'header':
2520.4.95 by Aaron Bentley
Add support for header/info records
262
                bytes = None
263
            else:
6634.2.1 by Martin
Apply 2to3 next fixer and make compatible
264
                _unused, bytes = next(iterator)
2682.1.1 by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings
265
            yield (bytes, metadata) + self.decode_name(names[0][0])
2520.4.22 by Aaron Bentley
Create ContainerReader
266
267
4237.3.1 by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in
268
class BundleSerializerV4(bundle_serializer.BundleSerializer):
2520.4.118 by Aaron Bentley
Add docs
269
    """Implement the high-level bundle interface"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
270
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
271
    def write_bundle(self, repository, target, base, fileobj):
2520.4.118 by Aaron Bentley
Add docs
272
        """Write a bundle to a file object
273
274
        :param repository: The repository to retrieve revision data from
275
        :param target: The head revision to include ancestors of
276
        :param base: The ancestor of the target to stop including acestors
277
            at.
278
        :param fileobj: The file-like object to write to
279
        """
7029.2.2 by Jelmer Vernooij
Fix two tests.
280
        write_op = BundleWriteOperation(base, target, repository, fileobj)
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
281
        return write_op.do_write()
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
282
283
    def read(self, file):
2520.4.118 by Aaron Bentley
Add docs
284
        """return a reader object for a given file"""
2520.4.72 by Aaron Bentley
Rename format to 4alpha
285
        bundle = BundleInfoV4(file, self)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
286
        return bundle
287
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
288
    @staticmethod
289
    def get_source_serializer(info):
2520.4.118 by Aaron Bentley
Add docs
290
        """Retrieve the serializer for a given info object"""
6973.7.10 by Jelmer Vernooij
More fixes.
291
        return serializer.format_registry.get(info[b'serializer'].decode('ascii'))
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
292
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
293
294
class BundleWriteOperation(object):
2520.4.118 by Aaron Bentley
Add docs
295
    """Perform the operation of writing revisions to a bundle"""
2520.4.123 by Aaron Bentley
Cleanup of bundle code
296
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
297
    def __init__(self, base, target, repository, fileobj, revision_ids=None):
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
298
        self.base = base
299
        self.target = target
300
        self.repository = repository
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
301
        bundle = BundleWriter(fileobj)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
302
        self.bundle = bundle
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
303
        if revision_ids is not None:
304
            self.revision_ids = revision_ids
305
        else:
4154.1.1 by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice
306
            graph = repository.get_graph()
4154.1.3 by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again
307
            revision_ids = graph.find_unique_ancestors(target, [base])
308
            # Strip ghosts
309
            parents = graph.get_parent_map(revision_ids)
310
            self.revision_ids = [r for r in revision_ids if r in parents]
6619.3.12 by Jelmer Vernooij
Use 2to3 set_literal fixer.
311
        self.revision_keys = {(revid,) for revid in self.revision_ids}
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
312
313
    def do_write(self):
2520.4.118 by Aaron Bentley
Add docs
314
        """Write all data to the bundle"""
6143.1.3 by Jonathan Riddell
more plurals
315
        trace.note(ngettext('Bundling %d revision.', 'Bundling %d revisions.',
316
                            len(self.revision_ids)), len(self.revision_ids))
6754.8.4 by Jelmer Vernooij
Use new context stuff.
317
        with self.repository.lock_read():
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
318
            self.bundle.begin()
319
            self.write_info()
320
            self.write_files()
321
            self.write_revisions()
322
            self.bundle.end()
2520.4.53 by Aaron Bentley
refactor bundle serialization to make write_bundle primary
323
        return self.revision_ids
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
324
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
325
    def write_info(self):
2520.4.118 by Aaron Bentley
Add docs
326
        """Write format info"""
2520.4.113 by Aaron Bentley
Avoid peeking at Repository._serializer
327
        serializer_format = self.repository.get_serializer_format()
2520.4.99 by Aaron Bentley
Test conversion across models
328
        supports_rich_root = {True: 1, False: 0}[
329
            self.repository.supports_rich_root()]
6973.7.8 by Jelmer Vernooij
Fix more tests.
330
        self.bundle.add_info_record({b'serializer': serializer_format,
331
                                     b'supports_rich_root': supports_rich_root})
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
332
2520.4.51 by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it
333
    def write_files(self):
2520.4.118 by Aaron Bentley
Add docs
334
        """Write bundle records for all revisions of all files"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
335
        text_keys = []
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
336
        altered_fileids = self.repository.fileids_altered_by_revision_ids(
7143.15.2 by Jelmer Vernooij
Run autopep8.
337
            self.revision_ids)
6656.1.1 by Martin
Apply 2to3 dict fixer and clean up resulting mess using view helpers
338
        for file_id, revision_ids in viewitems(altered_fileids):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
339
            for revision_id in revision_ids:
340
                text_keys.append((file_id, revision_id))
3350.6.10 by Martin Pool
VersionedFiles review cleanups
341
        self._add_mp_records_keys('file', self.repository.texts, text_keys)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
342
343
    def write_revisions(self):
2520.4.118 by Aaron Bentley
Add docs
344
        """Write bundle records for all revisions and signatures"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
345
        inv_vf = self.repository.inventories
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
346
        topological_order = [key[-1] for key in multiparent.topo_iter_keys(
7143.15.2 by Jelmer Vernooij
Run autopep8.
347
            inv_vf, self.revision_keys)]
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
348
        revision_order = topological_order
2520.4.75 by Aaron Bentley
Fix traceback on empty bundles.
349
        if self.target is not None and self.target in self.revision_ids:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
350
            # Make sure the target revision is always the last entry
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
351
            revision_order = list(topological_order)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
352
            revision_order.remove(self.target)
353
            revision_order.append(self.target)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
354
        if self.repository._serializer.support_altered_by_hack:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
355
            # Repositories that support_altered_by_hack means that
356
            # inventories.make_mpdiffs() contains all the data about the tree
357
            # shape. Formats without support_altered_by_hack require
358
            # chk_bytes/etc, so we use a different code path.
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
359
            self._add_mp_records_keys('inventory', inv_vf,
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
360
                                      [(revid,) for revid in topological_order])
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
361
        else:
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
362
            # Inventories should always be added in pure-topological order, so
363
            # that we can apply the mpdiff for the child to the parent texts.
4543.2.12 by John Arbash Meinel
Always sort the inventories in pure topological order
364
            self._add_inventory_mpdiffs_from_serializer(topological_order)
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
365
        self._add_revision_texts(revision_order)
366
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
367
    def _add_inventory_mpdiffs_from_serializer(self, revision_order):
4543.2.20 by John Arbash Meinel
Update from Martin's review feedback.
368
        """Generate mpdiffs by serializing inventories.
369
370
        The current repository only has part of the tree shape information in
371
        the 'inventories' vf. So we use serializer.write_inventory_to_string to
372
        get a 'full' representation of the tree shape, and then generate
373
        mpdiffs on that data stream. This stream can then be reconstructed on
374
        the other side.
375
        """
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
376
        inventory_key_order = [(r,) for r in revision_order]
5374.2.5 by John Arbash Meinel
Rework things a bit so the logic can be shared.
377
        generator = _MPDiffInventoryGenerator(self.repository,
378
                                              inventory_key_order)
379
        for revision_id, parent_ids, sha1, diff in generator.iter_diffs():
6973.7.10 by Jelmer Vernooij
More fixes.
380
            text = b''.join(diff.to_patch())
4543.2.4 by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig.
381
            self.bundle.add_multiparent_record(text, sha1, parent_ids,
382
                                               'inventory', revision_id, None)
383
4543.2.3 by John Arbash Meinel
Change the name to test_merge_directive
384
    def _add_revision_texts(self, revision_order):
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
385
        parent_map = self.repository.get_parent_map(revision_order)
7027.3.3 by Jelmer Vernooij
Add some more bees; support writing both bytes and unicode strings in build_tree_contents.
386
        revision_to_bytes = self.repository._serializer.write_revision_to_string
4202.3.1 by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together.
387
        revisions = self.repository.get_revisions(revision_order)
388
        for revision in revisions:
389
            revision_id = revision.revision_id
3099.3.5 by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly.
390
            parents = parent_map.get(revision_id, None)
7027.3.3 by Jelmer Vernooij
Add some more bees; support writing both bytes and unicode strings in build_tree_contents.
391
            revision_text = revision_to_bytes(revision)
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
392
            self.bundle.add_fulltext_record(revision_text, parents,
7143.15.2 by Jelmer Vernooij
Run autopep8.
393
                                            'revision', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
394
            try:
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
395
                self.bundle.add_fulltext_record(
396
                    self.repository.get_signature_text(
7143.15.2 by Jelmer Vernooij
Run autopep8.
397
                        revision_id), parents, 'signature', revision_id)
2520.4.34 by Aaron Bentley
Add signature support
398
            except errors.NoSuchRevision:
399
                pass
400
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
401
    @staticmethod
402
    def get_base_target(revision_ids, forced_bases, repository):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
403
        """Determine the base and target from old-style revision ids"""
2520.4.50 by Aaron Bentley
Split write functionality out into a separate object
404
        if len(revision_ids) == 0:
405
            return None, None
406
        target = revision_ids[0]
407
        base = forced_bases.get(target)
408
        if base is None:
409
            parents = repository.get_revision(target).parent_ids
410
            if len(parents) == 0:
411
                base = _mod_revision.NULL_REVISION
412
            else:
413
                base = parents[0]
414
        return base, target
415
3350.6.10 by Martin Pool
VersionedFiles review cleanups
416
    def _add_mp_records_keys(self, repo_kind, vf, keys):
2520.4.118 by Aaron Bentley
Add docs
417
        """Add multi-parent diff records to a bundle"""
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
418
        ordered_keys = list(multiparent.topo_iter_keys(vf, keys))
419
        mpdiffs = vf.make_mpdiffs(ordered_keys)
420
        sha1s = vf.get_sha1s(ordered_keys)
421
        parent_map = vf.get_parent_map(ordered_keys)
3350.8.3 by Robert Collins
VF.get_sha1s needed changing to be stackable.
422
        for mpdiff, item_key, in zip(mpdiffs, ordered_keys):
423
            sha1 = sha1s[item_key]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
424
            parents = [key[-1] for key in parent_map[item_key]]
6973.7.8 by Jelmer Vernooij
Fix more tests.
425
            text = b''.join(mpdiff.to_patch())
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
426
            # Infer file id records as appropriate.
427
            if len(item_key) == 2:
428
                file_id = item_key[0]
429
            else:
430
                file_id = None
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
431
            self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
432
                                               item_key[-1], file_id)
2520.4.6 by Aaron Bentley
Get installation started
433
434
2520.4.72 by Aaron Bentley
Rename format to 4alpha
435
class BundleInfoV4(object):
2520.4.6 by Aaron Bentley
Get installation started
436
2520.4.118 by Aaron Bentley
Add docs
437
    """Provide (most of) the BundleInfo interface"""
7143.15.2 by Jelmer Vernooij
Run autopep8.
438
2520.4.6 by Aaron Bentley
Get installation started
439
    def __init__(self, fileobj, serializer):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
440
        self._fileobj = fileobj
441
        self._serializer = serializer
442
        self.__real_revisions = None
443
        self.__revisions = None
444
445
    def install(self, repository):
446
        return self.install_revisions(repository)
447
2520.4.148 by Aaron Bentley
Updates from review
448
    def install_revisions(self, repository, stream_input=True):
449
        """Install this bundle's revisions into the specified repository
450
451
        :param target_repo: The repository to install into
452
        :param stream_input: If True, will stream input rather than reading it
453
            all into memory at once.  Reading it into memory all at once is
454
            (currently) faster.
455
        """
6973.11.4 by Jelmer Vernooij
use context managers.
456
        with repository.lock_write():
2520.4.148 by Aaron Bentley
Updates from review
457
            ri = RevisionInstaller(self.get_bundle_reader(stream_input),
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
458
                                   self._serializer, repository)
2520.4.18 by Aaron Bentley
Generate mpdiffs for inventory
459
            return ri.install()
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
460
2520.4.109 by Aaron Bentley
start work on directive cherry-picking
461
    def get_merge_request(self, target_repo):
462
        """Provide data for performing a merge
463
464
        Returns suggested base, suggested target, and patch verification status
465
        """
466
        return None, self.target, 'inapplicable'
467
2520.4.148 by Aaron Bentley
Updates from review
468
    def get_bundle_reader(self, stream_input=True):
469
        """Return a new BundleReader for the associated bundle
470
471
        :param stream_input: If True, the BundleReader stream input rather than
472
            reading it all into memory at once.  Reading it into memory all at
473
            once is (currently) faster.
474
        """
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
475
        self._fileobj.seek(0)
2520.4.148 by Aaron Bentley
Updates from review
476
        return BundleReader(self._fileobj, stream_input)
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
477
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
478
    def _get_real_revisions(self):
479
        if self.__real_revisions is None:
480
            self.__real_revisions = []
2520.4.39 by Aaron Bentley
Rename container => bundle(reader) where appropriate
481
            bundle_reader = self.get_bundle_reader()
2520.4.102 by Aaron Bentley
rename parents to metadata
482
            for bytes, metadata, repo_kind, revision_id, file_id in \
7143.15.2 by Jelmer Vernooij
Run autopep8.
483
                    bundle_reader.iter_records():
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
484
                if repo_kind == 'info':
485
                    serializer =\
2520.4.102 by Aaron Bentley
rename parents to metadata
486
                        self._serializer.get_source_serializer(metadata)
2520.4.22 by Aaron Bentley
Create ContainerReader
487
                if repo_kind == 'revision':
2520.4.101 by Aaron Bentley
Use a registry to look up xml serializers by format
488
                    rev = serializer.read_revision_from_string(bytes)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
489
                    self.__real_revisions.append(rev)
490
        return self.__real_revisions
491
    real_revisions = property(_get_real_revisions)
492
493
    def _get_revisions(self):
494
        if self.__revisions is None:
495
            self.__revisions = []
496
            for revision in self.real_revisions:
2520.4.33 by Aaron Bentley
remove test dependencies on serialization minutia
497
                self.__revisions.append(
498
                    bundle_data.RevisionInfo.from_revision(revision))
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
499
        return self.__revisions
500
501
    revisions = property(_get_revisions)
502
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
503
    def _get_target(self):
504
        return self.revisions[-1].revision_id
505
506
    target = property(_get_target)
507
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
508
509
class RevisionInstaller(object):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
510
    """Installs revisions into a repository"""
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
511
2520.4.21 by Aaron Bentley
Finish turning ContainerWriter into a new layer
512
    def __init__(self, container, serializer, repository):
513
        self._container = container
2520.4.6 by Aaron Bentley
Get installation started
514
        self._serializer = serializer
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
515
        self._repository = repository
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
516
        self._info = None
2520.4.99 by Aaron Bentley
Test conversion across models
517
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
518
    def install(self):
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
519
        """Perform the installation.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
520
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
521
        Must be called with the Repository locked.
522
        """
7143.21.1 by Jelmer Vernooij
Add WriteGroup contextmanager.
523
        with _mod_repository.WriteGroup(self._repository):
524
            return self._install_in_write_group()
2592.4.1 by Martin Pool
RevisionInstaller now creates a write group for its work
525
2856.1.2 by Robert Collins
Review feedback.
526
    def _install_in_write_group(self):
2520.4.6 by Aaron Bentley
Get installation started
527
        current_file = None
528
        current_versionedfile = None
529
        pending_file_records = []
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
530
        inventory_vf = None
531
        pending_inventory_records = []
2520.4.8 by Aaron Bentley
Serialize inventory
532
        added_inv = set()
2520.4.29 by Aaron Bentley
Reactivate some testing, fix topo_iter
533
        target_revision = None
2520.4.58 by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind
534
        for bytes, metadata, repo_kind, revision_id, file_id in\
7143.15.2 by Jelmer Vernooij
Run autopep8.
535
                self._container.iter_records():
2520.4.97 by Aaron Bentley
Hack in support for inventory conversion
536
            if repo_kind == 'info':
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
537
                if self._info is not None:
538
                    raise AssertionError()
2520.4.123 by Aaron Bentley
Cleanup of bundle code
539
                self._handle_info(metadata)
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
540
            if (pending_file_records and
7143.15.2 by Jelmer Vernooij
Run autopep8.
541
                    (repo_kind, file_id) != ('file', current_file)):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
542
                # Flush the data for a single file - prevents memory
543
                # spiking due to buffering all files in memory.
544
                self._install_mp_records_keys(self._repository.texts,
7143.15.2 by Jelmer Vernooij
Run autopep8.
545
                                              pending_file_records)
2520.4.8 by Aaron Bentley
Serialize inventory
546
                current_file = None
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
547
                del pending_file_records[:]
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
548
            if len(pending_inventory_records) > 0 and repo_kind != 'inventory':
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
549
                self._install_inventory_records(pending_inventory_records)
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
550
                pending_inventory_records = []
551
            if repo_kind == 'inventory':
7143.15.2 by Jelmer Vernooij
Run autopep8.
552
                pending_inventory_records.append(
553
                    ((revision_id,), metadata, bytes))
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
554
            if repo_kind == 'revision':
555
                target_revision = revision_id
556
                self._install_revision(revision_id, metadata, bytes)
557
            if repo_kind == 'signature':
558
                self._install_signature(revision_id, metadata, bytes)
2520.4.22 by Aaron Bentley
Create ContainerReader
559
            if repo_kind == 'file':
2520.4.142 by Aaron Bentley
Clean up installation of inventory records
560
                current_file = file_id
7143.15.2 by Jelmer Vernooij
Run autopep8.
561
                pending_file_records.append(
562
                    ((file_id, revision_id), metadata, bytes))
563
        self._install_mp_records_keys(
564
            self._repository.texts, pending_file_records)
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
565
        return target_revision
2520.4.6 by Aaron Bentley
Get installation started
566
2520.4.123 by Aaron Bentley
Cleanup of bundle code
567
    def _handle_info(self, info):
568
        """Extract data from an info record"""
569
        self._info = info
570
        self._source_serializer = self._serializer.get_source_serializer(info)
6973.7.10 by Jelmer Vernooij
More fixes.
571
        if (info[b'supports_rich_root'] == 0 and
7143.15.2 by Jelmer Vernooij
Run autopep8.
572
                self._repository.supports_rich_root()):
2520.4.123 by Aaron Bentley
Cleanup of bundle code
573
            self.update_root = True
574
        else:
575
            self.update_root = False
576
2520.4.60 by Aaron Bentley
Add sha1 verification for mpdiffs
577
    def _install_mp_records(self, versionedfile, records):
2520.4.61 by Aaron Bentley
Do bulk insertion of records
578
        if len(records) == 0:
579
            return
580
        d_func = multiparent.MultiParent.from_patch
581
        vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
582
                      records if r not in versionedfile]
583
        versionedfile.add_mpdiffs(vf_records)
2520.4.8 by Aaron Bentley
Serialize inventory
584
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
585
    def _install_mp_records_keys(self, versionedfile, records):
586
        d_func = multiparent.MultiParent.from_patch
587
        vf_records = []
588
        for key, meta, text in records:
3350.6.7 by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where.
589
            # Adapt to tuple interface: A length two key is a file_id,
590
            # revision_id pair, a length 1 key is a
591
            # revision/signature/inventory. We need to do this because
592
            # the metadata extraction from the bundle has not yet been updated
593
            # to use the consistent tuple interface itself.
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
594
            if len(key) == 2:
595
                prefix = key[:1]
596
            else:
597
                prefix = ()
6973.7.10 by Jelmer Vernooij
More fixes.
598
            parents = [prefix + (parent,) for parent in meta[b'parents']]
599
            vf_records.append((key, parents, meta[b'sha1'], d_func(text)))
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
600
        versionedfile.add_mpdiffs(vf_records)
601
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
602
    def _get_parent_inventory_texts(self, inventory_text_cache,
603
                                    inventory_cache, parent_ids):
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
604
        cached_parent_texts = {}
605
        remaining_parent_ids = []
606
        for parent_id in parent_ids:
607
            p_text = inventory_text_cache.get(parent_id, None)
608
            if p_text is None:
609
                remaining_parent_ids.append(parent_id)
610
            else:
611
                cached_parent_texts[parent_id] = p_text
612
        ghosts = ()
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
613
        # TODO: Use inventory_cache to grab inventories we already have in
614
        #       memory
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
615
        if remaining_parent_ids:
616
            # first determine what keys are actually present in the local
617
            # inventories object (don't use revisions as they haven't been
618
            # installed yet.)
619
            parent_keys = [(r,) for r in remaining_parent_ids]
620
            present_parent_map = self._repository.inventories.get_parent_map(
7143.15.2 by Jelmer Vernooij
Run autopep8.
621
                parent_keys)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
622
            present_parent_ids = []
623
            ghosts = set()
624
            for p_id in remaining_parent_ids:
625
                if (p_id,) in present_parent_map:
626
                    present_parent_ids.append(p_id)
627
                else:
628
                    ghosts.add(p_id)
629
            to_string = self._source_serializer.write_inventory_to_string
630
            for parent_inv in self._repository.iter_inventories(
7143.15.2 by Jelmer Vernooij
Run autopep8.
631
                    present_parent_ids):
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
632
                p_text = to_string(parent_inv)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
633
                inventory_cache[parent_inv.revision_id] = parent_inv
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
634
                cached_parent_texts[parent_inv.revision_id] = p_text
635
                inventory_text_cache[parent_inv.revision_id] = p_text
636
637
        parent_texts = [cached_parent_texts[parent_id]
638
                        for parent_id in parent_ids
7143.15.2 by Jelmer Vernooij
Run autopep8.
639
                        if parent_id not in ghosts]
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
640
        return parent_texts
641
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
642
    def _install_inventory_records(self, records):
6973.7.10 by Jelmer Vernooij
More fixes.
643
        if (self._info[b'serializer'] == self._repository._serializer.format_num
7143.15.2 by Jelmer Vernooij
Run autopep8.
644
                and self._repository._serializer.support_altered_by_hack):
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
645
            return self._install_mp_records_keys(self._repository.inventories,
7143.15.2 by Jelmer Vernooij
Run autopep8.
646
                                                 records)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
647
        # Use a 10MB text cache, since these are string xml inventories. Note
648
        # that 10MB is fairly small for large projects (a single inventory can
649
        # be >5MB). Another possibility is to cache 10-20 inventory texts
650
        # instead
7143.15.2 by Jelmer Vernooij
Run autopep8.
651
        inventory_text_cache = lru_cache.LRUSizeCache(10 * 1024 * 1024)
4543.2.21 by John Arbash Meinel
A few more tiny tweaks to comments, etc.
652
        # Also cache the in-memory representation. This allows us to create
653
        # inventory deltas to apply rather than calling add_inventory from
654
        # scratch each time.
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
655
        inventory_cache = lru_cache.LRUCache(10)
6861.4.1 by Jelmer Vernooij
Make progress bars context managers.
656
        with ui.ui_factory.nested_progress_bar() as pb:
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
657
            num_records = len(records)
658
            for idx, (key, metadata, bytes) in enumerate(records):
659
                pb.update('installing inventory', idx, num_records)
660
                revision_id = key[-1]
6973.7.10 by Jelmer Vernooij
More fixes.
661
                parent_ids = metadata[b'parents']
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
662
                # Note: This assumes the local ghosts are identical to the
663
                #       ghosts in the source, as the Bundle serialization
664
                #       format doesn't record ghosts.
665
                p_texts = self._get_parent_inventory_texts(inventory_text_cache,
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
666
                                                           inventory_cache,
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
667
                                                           parent_ids)
668
                # Why does to_lines() take strings as the source, it seems that
669
                # it would have to cast to a list of lines, which we get back
670
                # as lines and then cast back to a string.
671
                target_lines = multiparent.MultiParent.from_patch(bytes
7143.15.2 by Jelmer Vernooij
Run autopep8.
672
                                                                  ).to_lines(p_texts)
6973.7.10 by Jelmer Vernooij
More fixes.
673
                inv_text = b''.join(target_lines)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
674
                del target_lines
675
                sha1 = osutils.sha_string(inv_text)
6973.7.10 by Jelmer Vernooij
More fixes.
676
                if sha1 != metadata[b'sha1']:
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
677
                    raise errors.BadBundle("Can't convert to target format")
678
                # Add this to the cache so we don't have to extract it again.
679
                inventory_text_cache[revision_id] = inv_text
680
                target_inv = self._source_serializer.read_inventory_from_string(
681
                    inv_text)
682
                self._handle_root(target_inv, parent_ids)
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
683
                parent_inv = None
684
                if parent_ids:
685
                    parent_inv = inventory_cache.get(parent_ids[0], None)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
686
                try:
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
687
                    if parent_inv is None:
688
                        self._repository.add_inventory(revision_id, target_inv,
689
                                                       parent_ids)
690
                    else:
691
                        delta = target_inv._make_delta(parent_inv)
692
                        self._repository.add_inventory_by_delta(parent_ids[0],
7143.15.2 by Jelmer Vernooij
Run autopep8.
693
                                                                delta, revision_id, parent_ids)
4543.2.16 by John Arbash Meinel
Adding an inventory text cache.
694
                except errors.UnsupportedInventoryKind:
695
                    raise errors.IncompatibleRevision(repr(self._repository))
4543.2.17 by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta.
696
                inventory_cache[revision_id] = target_inv
2520.4.99 by Aaron Bentley
Test conversion across models
697
698
    def _handle_root(self, target_inv, parent_ids):
699
        revision_id = target_inv.revision_id
700
        if self.update_root:
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
701
            text_key = (target_inv.root.file_id, revision_id)
702
            parent_keys = [(target_inv.root.file_id, parent) for
7143.15.2 by Jelmer Vernooij
Run autopep8.
703
                           parent in parent_ids]
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
704
            self._repository.texts.add_lines(text_key, parent_keys, [])
2520.4.99 by Aaron Bentley
Test conversion across models
705
        elif not self._repository.supports_rich_root():
706
            if target_inv.root.revision != revision_id:
707
                raise errors.IncompatibleRevision(repr(self._repository))
708
2520.4.59 by Aaron Bentley
Push metadata down the stack
709
    def _install_revision(self, revision_id, metadata, text):
2520.4.14 by Aaron Bentley
Get most tests passing, use format header
710
        if self._repository.has_revision(revision_id):
711
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
712
        revision = self._source_serializer.read_revision_from_string(text)
713
        self._repository.add_revision(revision.revision_id, revision)
2520.4.34 by Aaron Bentley
Add signature support
714
2520.4.59 by Aaron Bentley
Push metadata down the stack
715
    def _install_signature(self, revision_id, metadata, text):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
716
        transaction = self._repository.get_transaction()
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
717
        if self._repository.has_signature_for_revision_id(revision_id):
2520.4.100 by Aaron Bentley
Fix repeat signature installs
718
            return
3350.6.4 by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
719
        self._repository.add_signature_text(revision_id, text)