bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
1 |
# Copyright (C) 2007-2010 Canonical Ltd
|
2520.4.85
by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!) |
2 |
#
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
#
|
|
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
#
|
|
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
4183.7.1
by Sabin Iacob
update FSF mailing address |
15 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
2520.4.85
by Aaron Bentley
Get all test passing (which just proves there aren't enough tests!) |
16 |
|
6379.6.3
by Jelmer Vernooij
Use absolute_import. |
17 |
from __future__ import absolute_import |
18 |
||
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
19 |
import bz2 |
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
20 |
import re |
2520.4.20
by Aaron Bentley
Compress and base64-encode bundle contents |
21 |
|
6624
by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes') |
22 |
from ... import ( |
6621.22.2
by Martin
Use BytesIO or StringIO from bzrlib.sixish |
23 |
bencode, |
2520.4.34
by Aaron Bentley
Add signature support |
24 |
errors, |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
25 |
iterablefile, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
26 |
lru_cache, |
2520.4.13
by Aaron Bentley
Use real container implementation |
27 |
multiparent, |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
28 |
osutils, |
2520.4.40
by Aaron Bentley
Add human-readable diff to bundles |
29 |
revision as _mod_revision, |
2520.4.45
by Aaron Bentley
Handle inconsistencies in last-modified-revision between vf and inventory |
30 |
trace, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
31 |
ui, |
6670.4.1
by Jelmer Vernooij
Update imports. |
32 |
)
|
33 |
from ...bzr import ( |
|
34 |
pack, |
|
6670.4.10
by Jelmer Vernooij
Move serializer to bzr. |
35 |
serializer, |
5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
36 |
versionedfile as _mod_versionedfile, |
2520.4.13
by Aaron Bentley
Use real container implementation |
37 |
)
|
6624
by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes') |
38 |
from ...bundle import bundle_data, serializer as bundle_serializer |
39 |
from ...i18n import ngettext |
|
40 |
from ...sixish import ( |
|
6621.22.2
by Martin
Use BytesIO or StringIO from bzrlib.sixish |
41 |
BytesIO, |
6656.1.1
by Martin
Apply 2to3 dict fixer and clean up resulting mess using view helpers |
42 |
viewitems, |
6621.22.2
by Martin
Use BytesIO or StringIO from bzrlib.sixish |
43 |
)
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
44 |
|
2520.4.4
by Aaron Bentley
Get basis support for a new bundle format in place |
45 |
|
5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
46 |
class _MPDiffInventoryGenerator(_mod_versionedfile._MPDiffGenerator): |
47 |
"""Generate Inventory diffs serialized inventories.""" |
|
48 |
||
49 |
def __init__(self, repo, inventory_keys): |
|
50 |
super(_MPDiffInventoryGenerator, self).__init__(repo.inventories, |
|
51 |
inventory_keys) |
|
52 |
self.repo = repo |
|
53 |
self.sha1s = {} |
|
54 |
||
55 |
def iter_diffs(self): |
|
56 |
"""Compute the diffs one at a time.""" |
|
57 |
# This is instead of compute_diffs() since we guarantee our ordering of
|
|
58 |
# inventories, we don't have to do any buffering
|
|
59 |
self._find_needed_keys() |
|
60 |
# We actually use a slightly different ordering. We grab all of the
|
|
61 |
# parents first, and then grab the ordered requests.
|
|
62 |
needed_ids = [k[-1] for k in self.present_parents] |
|
63 |
needed_ids.extend([k[-1] for k in self.ordered_keys]) |
|
64 |
inv_to_str = self.repo._serializer.write_inventory_to_string |
|
65 |
for inv in self.repo.iter_inventories(needed_ids): |
|
66 |
revision_id = inv.revision_id |
|
67 |
key = (revision_id,) |
|
68 |
if key in self.present_parents: |
|
69 |
# Not a key we will transmit, which is a shame, since because
|
|
70 |
# of that bundles don't work with stacked branches
|
|
71 |
parent_ids = None |
|
72 |
else: |
|
73 |
parent_ids = [k[-1] for k in self.parent_map[key]] |
|
74 |
as_bytes = inv_to_str(inv) |
|
75 |
self._process_one_record(key, (as_bytes,)) |
|
76 |
if parent_ids is None: |
|
77 |
continue
|
|
78 |
diff = self.diffs.pop(key) |
|
79 |
sha1 = osutils.sha_string(as_bytes) |
|
80 |
yield revision_id, parent_ids, sha1, diff |
|
81 |
||
82 |
||
2520.4.25
by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader |
83 |
class BundleWriter(object): |
2520.4.118
by Aaron Bentley
Add docs |
84 |
"""Writer for bundle-format files. |
85 |
||
86 |
This serves roughly the same purpose as ContainerReader, but acts as a
|
|
87 |
layer on top of it.
|
|
88 |
||
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
89 |
Provides ways of writing the specific record types supported this bundle
|
2520.4.118
by Aaron Bentley
Add docs |
90 |
format.
|
91 |
"""
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
92 |
|
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
93 |
def __init__(self, fileobj): |
2520.4.27
by Aaron Bentley
Use less memory when writing bzip-encoded files |
94 |
self._container = pack.ContainerWriter(self._write_encoded) |
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
95 |
self._fileobj = fileobj |
2520.4.27
by Aaron Bentley
Use less memory when writing bzip-encoded files |
96 |
self._compressor = bz2.BZ2Compressor() |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
97 |
|
2520.4.118
by Aaron Bentley
Add docs |
98 |
def _write_encoded(self, bytes): |
99 |
"""Write bzip2-encoded bytes to the file""" |
|
100 |
self._fileobj.write(self._compressor.compress(bytes)) |
|
101 |
||
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
102 |
def begin(self): |
2520.4.118
by Aaron Bentley
Add docs |
103 |
"""Start writing the bundle""" |
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
104 |
self._fileobj.write(bundle_serializer._get_bundle_header( |
105 |
bundle_serializer.v4_string)) |
|
2520.4.24
by Aaron Bentley
Move heading writing above container beginning |
106 |
self._fileobj.write('#\n') |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
107 |
self._container.begin() |
108 |
||
109 |
def end(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
110 |
"""Finish writing the bundle""" |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
111 |
self._container.end() |
2520.4.76
by Aaron Bentley
Move base64-encoding into merge directives |
112 |
self._fileobj.write(self._compressor.flush()) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
113 |
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
114 |
def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind, |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
115 |
revision_id, file_id): |
2520.4.118
by Aaron Bentley
Add docs |
116 |
"""Add a record for a multi-parent diff |
117 |
||
118 |
:mp_bytes: A multi-parent diff, as a bytestring
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
119 |
:sha1: The sha1 hash of the fulltext
|
2520.4.118
by Aaron Bentley
Add docs |
120 |
:parents: a list of revision-ids of the parents
|
121 |
:repo_kind: The kind of object in the repository. May be 'file' or
|
|
122 |
'inventory'
|
|
123 |
:revision_id: The revision id of the mpdiff being added.
|
|
124 |
:file_id: The file-id of the file, or None for inventories.
|
|
125 |
"""
|
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
126 |
metadata = {'parents': parents, |
127 |
'storage_kind': 'mpdiff', |
|
128 |
'sha1': sha1} |
|
129 |
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id) |
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
130 |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
131 |
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id): |
2520.4.118
by Aaron Bentley
Add docs |
132 |
"""Add a record for a fulltext |
133 |
||
134 |
:bytes: The fulltext, as a bytestring
|
|
135 |
:parents: a list of revision-ids of the parents
|
|
136 |
:repo_kind: The kind of object in the repository. May be 'revision' or
|
|
137 |
'signature'
|
|
138 |
:revision_id: The revision id of the fulltext being added.
|
|
139 |
"""
|
|
140 |
metadata = {'parents': parents, |
|
2520.5.3
by Aaron Bentley
fix sha1 in bundle format 4 |
141 |
'storage_kind': 'mpdiff'} |
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
142 |
self._add_record(bytes, {'parents': parents, |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
143 |
'storage_kind': 'fulltext'}, repo_kind, revision_id, None) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
144 |
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
145 |
def add_info_record(self, **kwargs): |
2520.4.118
by Aaron Bentley
Add docs |
146 |
"""Add an info record to the bundle |
147 |
||
148 |
Any parameters may be supplied, except 'self' and 'storage_kind'.
|
|
149 |
Values must be lists, strings, integers, dicts, or a combination.
|
|
150 |
"""
|
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
151 |
kwargs['storage_kind'] = 'header' |
152 |
self._add_record(None, kwargs, 'info', None, None) |
|
153 |
||
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
154 |
@staticmethod
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
155 |
def encode_name(content_kind, revision_id, file_id=None): |
2520.4.118
by Aaron Bentley
Add docs |
156 |
"""Encode semantic ids as a container name""" |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
157 |
if content_kind not in ('revision', 'file', 'inventory', 'signature', |
158 |
'info'): |
|
159 |
raise ValueError(content_kind) |
|
2520.4.118
by Aaron Bentley
Add docs |
160 |
if content_kind == 'file': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
161 |
if file_id is None: |
162 |
raise AssertionError() |
|
2520.4.118
by Aaron Bentley
Add docs |
163 |
else: |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
164 |
if file_id is not None: |
165 |
raise AssertionError() |
|
2520.4.95
by Aaron Bentley
Add support for header/info records |
166 |
if content_kind == 'info': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
167 |
if revision_id is not None: |
168 |
raise AssertionError() |
|
169 |
elif revision_id is None: |
|
170 |
raise AssertionError() |
|
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
171 |
names = [n.replace('/', '//') for n in |
172 |
(content_kind, revision_id, file_id) if n is not None] |
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
173 |
return '/'.join(names) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
174 |
|
2520.4.56
by Aaron Bentley
Begin adding support for arbitrary metadata |
175 |
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id): |
2520.4.118
by Aaron Bentley
Add docs |
176 |
"""Add a bundle record to the container. |
177 |
||
178 |
Most bundle records are recorded as header/body pairs, with the
|
|
179 |
body being nameless. Records with storage_kind 'header' have no
|
|
180 |
body.
|
|
181 |
"""
|
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
182 |
name = self.encode_name(repo_kind, revision_id, file_id) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
183 |
encoded_metadata = bencode.bencode(metadata) |
2682.1.1
by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings |
184 |
self._container.add_bytes_record(encoded_metadata, [(name, )]) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
185 |
if metadata['storage_kind'] != 'header': |
186 |
self._container.add_bytes_record(bytes, []) |
|
2520.4.13
by Aaron Bentley
Use real container implementation |
187 |
|
2520.4.7
by Aaron Bentley
Fix patch deserialization |
188 |
|
2520.4.25
by Aaron Bentley
Rename ContainerWriter/ContainerReader to BundleWriter/BundleReader |
189 |
class BundleReader(object): |
2520.4.118
by Aaron Bentley
Add docs |
190 |
"""Reader for bundle-format files. |
191 |
||
192 |
This serves roughly the same purpose as ContainerReader, but acts as a
|
|
193 |
layer on top of it, providing metadata, a semantic name, and a record
|
|
194 |
body
|
|
195 |
"""
|
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
196 |
|
4543.2.14
by John Arbash Meinel
Clarify some comments, fix up a debugging change. |
197 |
def __init__(self, fileobj, stream_input=True): |
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
198 |
"""Constructor |
199 |
||
200 |
:param fileobj: a file containing a bzip-encoded container
|
|
2520.4.148
by Aaron Bentley
Updates from review |
201 |
:param stream_input: If True, the BundleReader stream input rather than
|
202 |
reading it all into memory at once. Reading it into memory all at
|
|
203 |
once is (currently) faster.
|
|
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
204 |
"""
|
2520.4.23
by Aaron Bentley
Move responsability for encoding into container objects |
205 |
line = fileobj.readline() |
206 |
if line != '\n': |
|
207 |
fileobj.readline() |
|
2520.4.40
by Aaron Bentley
Add human-readable diff to bundles |
208 |
self.patch_lines = [] |
2520.4.148
by Aaron Bentley
Updates from review |
209 |
if stream_input: |
2520.4.145
by Aaron Bentley
Add memory_friendly toggle, be memory-unfriendly for merge directives |
210 |
source_file = iterablefile.IterableFile(self.iter_decode(fileobj)) |
211 |
else: |
|
6621.22.2
by Martin
Use BytesIO or StringIO from bzrlib.sixish |
212 |
source_file = BytesIO(bz2.decompress(fileobj.read())) |
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
213 |
self._container_file = source_file |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
214 |
|
215 |
@staticmethod
|
|
216 |
def iter_decode(fileobj): |
|
2520.4.118
by Aaron Bentley
Add docs |
217 |
"""Iterate through decoded fragments of the file""" |
2520.4.26
by Aaron Bentley
Make decompression reasonably memory-efficient |
218 |
decompressor = bz2.BZ2Decompressor() |
219 |
for line in fileobj: |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
220 |
try: |
221 |
yield decompressor.decompress(line) |
|
222 |
except EOFError: |
|
223 |
return
|
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
224 |
|
225 |
@staticmethod
|
|
226 |
def decode_name(name): |
|
2520.4.118
by Aaron Bentley
Add docs |
227 |
"""Decode a name from its container form into a semantic form |
228 |
||
229 |
:retval: content_kind, revision_id, file_id
|
|
230 |
"""
|
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
231 |
segments = re.split('(//?)', name) |
232 |
names = [''] |
|
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
233 |
for segment in segments: |
234 |
if segment == '//': |
|
235 |
names[-1] += '/' |
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
236 |
elif segment == '/': |
2520.4.127
by Aaron Bentley
Fix up name encoding to handle revision-ids with slashes |
237 |
names.append('') |
238 |
else: |
|
239 |
names[-1] += segment |
|
2520.4.130
by Aaron Bentley
Finish tweaking decode_name |
240 |
content_kind = names[0] |
2520.4.95
by Aaron Bentley
Add support for header/info records |
241 |
revision_id = None |
242 |
file_id = None |
|
243 |
if len(names) > 1: |
|
244 |
revision_id = names[1] |
|
2520.4.68
by Aaron Bentley
Change name separators to all-slash |
245 |
if len(names) > 2: |
246 |
file_id = names[2] |
|
247 |
return content_kind, revision_id, file_id |
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
248 |
|
249 |
def iter_records(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
250 |
"""Iterate through bundle records |
251 |
||
252 |
:return: a generator of (bytes, metadata, content_kind, revision_id,
|
|
253 |
file_id)
|
|
254 |
"""
|
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
255 |
iterator = pack.iter_records_from_file(self._container_file) |
256 |
for names, bytes in iterator: |
|
2520.4.131
by Aaron Bentley
Raise BadBundle for records with wrong number of names |
257 |
if len(names) != 1: |
258 |
raise errors.BadBundle('Record has %d names instead of 1' |
|
259 |
% len(names)) |
|
2916.2.18
by Andrew Bennetts
Use iter_records_from_file rather than ContainerReader. |
260 |
metadata = bencode.bdecode(bytes) |
2520.4.95
by Aaron Bentley
Add support for header/info records |
261 |
if metadata['storage_kind'] == 'header': |
262 |
bytes = None |
|
263 |
else: |
|
6634.2.1
by Martin
Apply 2to3 next fixer and make compatible |
264 |
_unused, bytes = next(iterator) |
2682.1.1
by Robert Collins
* The ``bzrlib.pack`` interface has changed to use tuples of bytestrings |
265 |
yield (bytes, metadata) + self.decode_name(names[0][0]) |
2520.4.22
by Aaron Bentley
Create ContainerReader |
266 |
|
267 |
||
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
268 |
class BundleSerializerV4(bundle_serializer.BundleSerializer): |
2520.4.118
by Aaron Bentley
Add docs |
269 |
"""Implement the high-level bundle interface""" |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
270 |
|
2520.4.4
by Aaron Bentley
Get basis support for a new bundle format in place |
271 |
def write(self, repository, revision_ids, forced_bases, fileobj): |
2520.4.118
by Aaron Bentley
Add docs |
272 |
"""Write a bundle to a file-like object |
273 |
||
274 |
For backwards-compatibility only
|
|
275 |
"""
|
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
276 |
write_op = BundleWriteOperation.from_old_args(repository, revision_ids, |
277 |
forced_bases, fileobj) |
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
278 |
return write_op.do_write() |
279 |
||
280 |
def write_bundle(self, repository, target, base, fileobj): |
|
2520.4.118
by Aaron Bentley
Add docs |
281 |
"""Write a bundle to a file object |
282 |
||
283 |
:param repository: The repository to retrieve revision data from
|
|
284 |
:param target: The head revision to include ancestors of
|
|
285 |
:param base: The ancestor of the target to stop including acestors
|
|
286 |
at.
|
|
287 |
:param fileobj: The file-like object to write to
|
|
288 |
"""
|
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
289 |
write_op = BundleWriteOperation(base, target, repository, fileobj) |
290 |
return write_op.do_write() |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
291 |
|
292 |
def read(self, file): |
|
2520.4.118
by Aaron Bentley
Add docs |
293 |
"""return a reader object for a given file""" |
2520.4.72
by Aaron Bentley
Rename format to 4alpha |
294 |
bundle = BundleInfoV4(file, self) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
295 |
return bundle |
296 |
||
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
297 |
@staticmethod
|
298 |
def get_source_serializer(info): |
|
2520.4.118
by Aaron Bentley
Add docs |
299 |
"""Retrieve the serializer for a given info object""" |
4237.3.1
by Jelmer Vernooij
Add new module with generic serializer information; keep XML-specific bits in |
300 |
return serializer.format_registry.get(info['serializer']) |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
301 |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
302 |
|
303 |
class BundleWriteOperation(object): |
|
2520.4.118
by Aaron Bentley
Add docs |
304 |
"""Perform the operation of writing revisions to a bundle""" |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
305 |
|
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
306 |
@classmethod
|
307 |
def from_old_args(cls, repository, revision_ids, forced_bases, fileobj): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
308 |
"""Create a BundleWriteOperation from old-style arguments""" |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
309 |
base, target = cls.get_base_target(revision_ids, forced_bases, |
310 |
repository) |
|
311 |
return BundleWriteOperation(base, target, repository, fileobj, |
|
312 |
revision_ids) |
|
313 |
||
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
314 |
def __init__(self, base, target, repository, fileobj, revision_ids=None): |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
315 |
self.base = base |
316 |
self.target = target |
|
317 |
self.repository = repository |
|
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
318 |
bundle = BundleWriter(fileobj) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
319 |
self.bundle = bundle |
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
320 |
if revision_ids is not None: |
321 |
self.revision_ids = revision_ids |
|
322 |
else: |
|
4154.1.1
by Ian Clatworthy
make send use graph.find_difference() instead of walking all of history twice |
323 |
graph = repository.get_graph() |
4154.1.3
by Ian Clatworthy
strip ghosts so test_bundle_with_ghosts works again |
324 |
revision_ids = graph.find_unique_ancestors(target, [base]) |
325 |
# Strip ghosts
|
|
326 |
parents = graph.get_parent_map(revision_ids) |
|
327 |
self.revision_ids = [r for r in revision_ids if r in parents] |
|
6619.3.12
by Jelmer Vernooij
Use 2to3 set_literal fixer. |
328 |
self.revision_keys = {(revid,) for revid in self.revision_ids} |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
329 |
|
330 |
def do_write(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
331 |
"""Write all data to the bundle""" |
6143.1.3
by Jonathan Riddell
more plurals |
332 |
trace.note(ngettext('Bundling %d revision.', 'Bundling %d revisions.', |
333 |
len(self.revision_ids)), len(self.revision_ids)) |
|
6754.8.4
by Jelmer Vernooij
Use new context stuff. |
334 |
with self.repository.lock_read(): |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
335 |
self.bundle.begin() |
336 |
self.write_info() |
|
337 |
self.write_files() |
|
338 |
self.write_revisions() |
|
339 |
self.bundle.end() |
|
2520.4.53
by Aaron Bentley
refactor bundle serialization to make write_bundle primary |
340 |
return self.revision_ids |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
341 |
|
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
342 |
def write_info(self): |
2520.4.118
by Aaron Bentley
Add docs |
343 |
"""Write format info""" |
2520.4.113
by Aaron Bentley
Avoid peeking at Repository._serializer |
344 |
serializer_format = self.repository.get_serializer_format() |
2520.4.99
by Aaron Bentley
Test conversion across models |
345 |
supports_rich_root = {True: 1, False: 0}[ |
346 |
self.repository.supports_rich_root()] |
|
2520.4.113
by Aaron Bentley
Avoid peeking at Repository._serializer |
347 |
self.bundle.add_info_record(serializer=serializer_format, |
2520.4.99
by Aaron Bentley
Test conversion across models |
348 |
supports_rich_root=supports_rich_root) |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
349 |
|
2520.4.51
by Aaron Bentley
Split iteration through file revisions into a method, so we can vary it |
350 |
def write_files(self): |
2520.4.118
by Aaron Bentley
Add docs |
351 |
"""Write bundle records for all revisions of all files""" |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
352 |
text_keys = [] |
3350.6.7
by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where. |
353 |
altered_fileids = self.repository.fileids_altered_by_revision_ids( |
354 |
self.revision_ids) |
|
6656.1.1
by Martin
Apply 2to3 dict fixer and clean up resulting mess using view helpers |
355 |
for file_id, revision_ids in viewitems(altered_fileids): |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
356 |
for revision_id in revision_ids: |
357 |
text_keys.append((file_id, revision_id)) |
|
3350.6.10
by Martin Pool
VersionedFiles review cleanups |
358 |
self._add_mp_records_keys('file', self.repository.texts, text_keys) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
359 |
|
360 |
def write_revisions(self): |
|
2520.4.118
by Aaron Bentley
Add docs |
361 |
"""Write bundle records for all revisions and signatures""" |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
362 |
inv_vf = self.repository.inventories |
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
363 |
topological_order = [key[-1] for key in multiparent.topo_iter_keys( |
364 |
inv_vf, self.revision_keys)] |
|
365 |
revision_order = topological_order |
|
2520.4.75
by Aaron Bentley
Fix traceback on empty bundles. |
366 |
if self.target is not None and self.target in self.revision_ids: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
367 |
# Make sure the target revision is always the last entry
|
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
368 |
revision_order = list(topological_order) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
369 |
revision_order.remove(self.target) |
370 |
revision_order.append(self.target) |
|
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
371 |
if self.repository._serializer.support_altered_by_hack: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
372 |
# Repositories that support_altered_by_hack means that
|
373 |
# inventories.make_mpdiffs() contains all the data about the tree
|
|
374 |
# shape. Formats without support_altered_by_hack require
|
|
375 |
# chk_bytes/etc, so we use a different code path.
|
|
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
376 |
self._add_mp_records_keys('inventory', inv_vf, |
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
377 |
[(revid,) for revid in topological_order]) |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
378 |
else: |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
379 |
# Inventories should always be added in pure-topological order, so
|
380 |
# that we can apply the mpdiff for the child to the parent texts.
|
|
4543.2.12
by John Arbash Meinel
Always sort the inventories in pure topological order |
381 |
self._add_inventory_mpdiffs_from_serializer(topological_order) |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
382 |
self._add_revision_texts(revision_order) |
383 |
||
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
384 |
def _add_inventory_mpdiffs_from_serializer(self, revision_order): |
4543.2.20
by John Arbash Meinel
Update from Martin's review feedback. |
385 |
"""Generate mpdiffs by serializing inventories. |
386 |
||
387 |
The current repository only has part of the tree shape information in
|
|
388 |
the 'inventories' vf. So we use serializer.write_inventory_to_string to
|
|
389 |
get a 'full' representation of the tree shape, and then generate
|
|
390 |
mpdiffs on that data stream. This stream can then be reconstructed on
|
|
391 |
the other side.
|
|
392 |
"""
|
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
393 |
inventory_key_order = [(r,) for r in revision_order] |
5374.2.5
by John Arbash Meinel
Rework things a bit so the logic can be shared. |
394 |
generator = _MPDiffInventoryGenerator(self.repository, |
395 |
inventory_key_order) |
|
396 |
for revision_id, parent_ids, sha1, diff in generator.iter_diffs(): |
|
4543.2.4
by John Arbash Meinel
Start working on code that will use Repository._serializer.write_inventory_to_strig. |
397 |
text = ''.join(diff.to_patch()) |
398 |
self.bundle.add_multiparent_record(text, sha1, parent_ids, |
|
399 |
'inventory', revision_id, None) |
|
400 |
||
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
401 |
def _add_revision_texts(self, revision_order): |
3099.3.5
by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly. |
402 |
parent_map = self.repository.get_parent_map(revision_order) |
4202.3.1
by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together. |
403 |
revision_to_str = self.repository._serializer.write_revision_to_string |
404 |
revisions = self.repository.get_revisions(revision_order) |
|
405 |
for revision in revisions: |
|
406 |
revision_id = revision.revision_id |
|
3099.3.5
by John Arbash Meinel
Update the last couple of places that referred to Provider.get_parents() directly. |
407 |
parents = parent_map.get(revision_id, None) |
4202.3.1
by Andrew Bennetts
Don't use get_revision_xml when writing a bundle, instead get all the revisions together. |
408 |
revision_text = revision_to_str(revision) |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
409 |
self.bundle.add_fulltext_record(revision_text, parents, |
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
410 |
'revision', revision_id) |
2520.4.34
by Aaron Bentley
Add signature support |
411 |
try: |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
412 |
self.bundle.add_fulltext_record( |
413 |
self.repository.get_signature_text( |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
414 |
revision_id), parents, 'signature', revision_id) |
2520.4.34
by Aaron Bentley
Add signature support |
415 |
except errors.NoSuchRevision: |
416 |
pass
|
|
417 |
||
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
418 |
@staticmethod
|
419 |
def get_base_target(revision_ids, forced_bases, repository): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
420 |
"""Determine the base and target from old-style revision ids""" |
2520.4.50
by Aaron Bentley
Split write functionality out into a separate object |
421 |
if len(revision_ids) == 0: |
422 |
return None, None |
|
423 |
target = revision_ids[0] |
|
424 |
base = forced_bases.get(target) |
|
425 |
if base is None: |
|
426 |
parents = repository.get_revision(target).parent_ids |
|
427 |
if len(parents) == 0: |
|
428 |
base = _mod_revision.NULL_REVISION |
|
429 |
else: |
|
430 |
base = parents[0] |
|
431 |
return base, target |
|
432 |
||
3350.6.10
by Martin Pool
VersionedFiles review cleanups |
433 |
def _add_mp_records_keys(self, repo_kind, vf, keys): |
2520.4.118
by Aaron Bentley
Add docs |
434 |
"""Add multi-parent diff records to a bundle""" |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
435 |
ordered_keys = list(multiparent.topo_iter_keys(vf, keys)) |
436 |
mpdiffs = vf.make_mpdiffs(ordered_keys) |
|
437 |
sha1s = vf.get_sha1s(ordered_keys) |
|
438 |
parent_map = vf.get_parent_map(ordered_keys) |
|
3350.8.3
by Robert Collins
VF.get_sha1s needed changing to be stackable. |
439 |
for mpdiff, item_key, in zip(mpdiffs, ordered_keys): |
440 |
sha1 = sha1s[item_key] |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
441 |
parents = [key[-1] for key in parent_map[item_key]] |
2520.4.41
by Aaron Bentley
Accelerate mpdiff generation |
442 |
text = ''.join(mpdiff.to_patch()) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
443 |
# Infer file id records as appropriate.
|
444 |
if len(item_key) == 2: |
|
445 |
file_id = item_key[0] |
|
446 |
else: |
|
447 |
file_id = None |
|
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
448 |
self.bundle.add_multiparent_record(text, sha1, parents, repo_kind, |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
449 |
item_key[-1], file_id) |
2520.4.6
by Aaron Bentley
Get installation started |
450 |
|
451 |
||
2520.4.72
by Aaron Bentley
Rename format to 4alpha |
452 |
class BundleInfoV4(object): |
2520.4.6
by Aaron Bentley
Get installation started |
453 |
|
2520.4.118
by Aaron Bentley
Add docs |
454 |
"""Provide (most of) the BundleInfo interface""" |
2520.4.6
by Aaron Bentley
Get installation started |
455 |
def __init__(self, fileobj, serializer): |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
456 |
self._fileobj = fileobj |
457 |
self._serializer = serializer |
|
458 |
self.__real_revisions = None |
|
459 |
self.__revisions = None |
|
460 |
||
461 |
def install(self, repository): |
|
462 |
return self.install_revisions(repository) |
|
463 |
||
2520.4.148
by Aaron Bentley
Updates from review |
464 |
def install_revisions(self, repository, stream_input=True): |
465 |
"""Install this bundle's revisions into the specified repository |
|
466 |
||
467 |
:param target_repo: The repository to install into
|
|
468 |
:param stream_input: If True, will stream input rather than reading it
|
|
469 |
all into memory at once. Reading it into memory all at once is
|
|
470 |
(currently) faster.
|
|
471 |
"""
|
|
2520.4.18
by Aaron Bentley
Generate mpdiffs for inventory |
472 |
repository.lock_write() |
473 |
try: |
|
2520.4.148
by Aaron Bentley
Updates from review |
474 |
ri = RevisionInstaller(self.get_bundle_reader(stream_input), |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
475 |
self._serializer, repository) |
2520.4.18
by Aaron Bentley
Generate mpdiffs for inventory |
476 |
return ri.install() |
477 |
finally: |
|
478 |
repository.unlock() |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
479 |
|
2520.4.109
by Aaron Bentley
start work on directive cherry-picking |
480 |
def get_merge_request(self, target_repo): |
481 |
"""Provide data for performing a merge |
|
482 |
||
483 |
Returns suggested base, suggested target, and patch verification status
|
|
484 |
"""
|
|
485 |
return None, self.target, 'inapplicable' |
|
486 |
||
2520.4.148
by Aaron Bentley
Updates from review |
487 |
def get_bundle_reader(self, stream_input=True): |
488 |
"""Return a new BundleReader for the associated bundle |
|
489 |
||
490 |
:param stream_input: If True, the BundleReader stream input rather than
|
|
491 |
reading it all into memory at once. Reading it into memory all at
|
|
492 |
once is (currently) faster.
|
|
493 |
"""
|
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
494 |
self._fileobj.seek(0) |
2520.4.148
by Aaron Bentley
Updates from review |
495 |
return BundleReader(self._fileobj, stream_input) |
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
496 |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
497 |
def _get_real_revisions(self): |
498 |
if self.__real_revisions is None: |
|
499 |
self.__real_revisions = [] |
|
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
500 |
bundle_reader = self.get_bundle_reader() |
2520.4.102
by Aaron Bentley
rename parents to metadata |
501 |
for bytes, metadata, repo_kind, revision_id, file_id in \ |
2520.4.39
by Aaron Bentley
Rename container => bundle(reader) where appropriate |
502 |
bundle_reader.iter_records(): |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
503 |
if repo_kind == 'info': |
504 |
serializer =\ |
|
2520.4.102
by Aaron Bentley
rename parents to metadata |
505 |
self._serializer.get_source_serializer(metadata) |
2520.4.22
by Aaron Bentley
Create ContainerReader |
506 |
if repo_kind == 'revision': |
2520.4.101
by Aaron Bentley
Use a registry to look up xml serializers by format |
507 |
rev = serializer.read_revision_from_string(bytes) |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
508 |
self.__real_revisions.append(rev) |
509 |
return self.__real_revisions |
|
510 |
real_revisions = property(_get_real_revisions) |
|
511 |
||
512 |
def _get_revisions(self): |
|
513 |
if self.__revisions is None: |
|
514 |
self.__revisions = [] |
|
515 |
for revision in self.real_revisions: |
|
2520.4.33
by Aaron Bentley
remove test dependencies on serialization minutia |
516 |
self.__revisions.append( |
517 |
bundle_data.RevisionInfo.from_revision(revision)) |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
518 |
return self.__revisions |
519 |
||
520 |
revisions = property(_get_revisions) |
|
521 |
||
2520.4.29
by Aaron Bentley
Reactivate some testing, fix topo_iter |
522 |
def _get_target(self): |
523 |
return self.revisions[-1].revision_id |
|
524 |
||
525 |
target = property(_get_target) |
|
526 |
||
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
527 |
|
528 |
class RevisionInstaller(object): |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
529 |
"""Installs revisions into a repository""" |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
530 |
|
2520.4.21
by Aaron Bentley
Finish turning ContainerWriter into a new layer |
531 |
def __init__(self, container, serializer, repository): |
532 |
self._container = container |
|
2520.4.6
by Aaron Bentley
Get installation started |
533 |
self._serializer = serializer |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
534 |
self._repository = repository |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
535 |
self._info = None |
2520.4.99
by Aaron Bentley
Test conversion across models |
536 |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
537 |
def install(self): |
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
538 |
"""Perform the installation. |
3943.8.1
by Marius Kruger
remove all trailing whitespace from bzr source |
539 |
|
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
540 |
Must be called with the Repository locked.
|
541 |
"""
|
|
542 |
self._repository.start_write_group() |
|
543 |
try: |
|
2856.1.2
by Robert Collins
Review feedback. |
544 |
result = self._install_in_write_group() |
2592.4.1
by Martin Pool
RevisionInstaller now creates a write group for its work |
545 |
except: |
546 |
self._repository.abort_write_group() |
|
547 |
raise
|
|
548 |
self._repository.commit_write_group() |
|
549 |
return result |
|
550 |
||
2856.1.2
by Robert Collins
Review feedback. |
551 |
def _install_in_write_group(self): |
2520.4.6
by Aaron Bentley
Get installation started |
552 |
current_file = None |
553 |
current_versionedfile = None |
|
554 |
pending_file_records = [] |
|
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
555 |
inventory_vf = None |
556 |
pending_inventory_records = [] |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
557 |
added_inv = set() |
2520.4.29
by Aaron Bentley
Reactivate some testing, fix topo_iter |
558 |
target_revision = None |
2520.4.58
by Aaron Bentley
Propogate support for metadata to iter_revisions, add storage kind |
559 |
for bytes, metadata, repo_kind, revision_id, file_id in\ |
2520.4.22
by Aaron Bentley
Create ContainerReader |
560 |
self._container.iter_records(): |
2520.4.97
by Aaron Bentley
Hack in support for inventory conversion |
561 |
if repo_kind == 'info': |
3376.2.4
by Martin Pool
Remove every assert statement from bzrlib! |
562 |
if self._info is not None: |
563 |
raise AssertionError() |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
564 |
self._handle_info(metadata) |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
565 |
if (pending_file_records and |
566 |
(repo_kind, file_id) != ('file', current_file)): |
|
567 |
# Flush the data for a single file - prevents memory
|
|
568 |
# spiking due to buffering all files in memory.
|
|
569 |
self._install_mp_records_keys(self._repository.texts, |
|
570 |
pending_file_records) |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
571 |
current_file = None |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
572 |
del pending_file_records[:] |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
573 |
if len(pending_inventory_records) > 0 and repo_kind != 'inventory': |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
574 |
self._install_inventory_records(pending_inventory_records) |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
575 |
pending_inventory_records = [] |
576 |
if repo_kind == 'inventory': |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
577 |
pending_inventory_records.append(((revision_id,), metadata, bytes)) |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
578 |
if repo_kind == 'revision': |
579 |
target_revision = revision_id |
|
580 |
self._install_revision(revision_id, metadata, bytes) |
|
581 |
if repo_kind == 'signature': |
|
582 |
self._install_signature(revision_id, metadata, bytes) |
|
2520.4.22
by Aaron Bentley
Create ContainerReader |
583 |
if repo_kind == 'file': |
2520.4.142
by Aaron Bentley
Clean up installation of inventory records |
584 |
current_file = file_id |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
585 |
pending_file_records.append(((file_id, revision_id), metadata, bytes)) |
586 |
self._install_mp_records_keys(self._repository.texts, pending_file_records) |
|
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
587 |
return target_revision |
2520.4.6
by Aaron Bentley
Get installation started |
588 |
|
2520.4.123
by Aaron Bentley
Cleanup of bundle code |
589 |
def _handle_info(self, info): |
590 |
"""Extract data from an info record""" |
|
591 |
self._info = info |
|
592 |
self._source_serializer = self._serializer.get_source_serializer(info) |
|
593 |
if (info['supports_rich_root'] == 0 and |
|
594 |
self._repository.supports_rich_root()): |
|
595 |
self.update_root = True |
|
596 |
else: |
|
597 |
self.update_root = False |
|
598 |
||
2520.4.60
by Aaron Bentley
Add sha1 verification for mpdiffs |
599 |
def _install_mp_records(self, versionedfile, records): |
2520.4.61
by Aaron Bentley
Do bulk insertion of records |
600 |
if len(records) == 0: |
601 |
return
|
|
602 |
d_func = multiparent.MultiParent.from_patch |
|
603 |
vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in |
|
604 |
records if r not in versionedfile] |
|
605 |
versionedfile.add_mpdiffs(vf_records) |
|
2520.4.8
by Aaron Bentley
Serialize inventory |
606 |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
607 |
def _install_mp_records_keys(self, versionedfile, records): |
608 |
d_func = multiparent.MultiParent.from_patch |
|
609 |
vf_records = [] |
|
610 |
for key, meta, text in records: |
|
3350.6.7
by Robert Collins
Review feedback, making things more clear, adding documentation on what is used where. |
611 |
# Adapt to tuple interface: A length two key is a file_id,
|
612 |
# revision_id pair, a length 1 key is a
|
|
613 |
# revision/signature/inventory. We need to do this because
|
|
614 |
# the metadata extraction from the bundle has not yet been updated
|
|
615 |
# to use the consistent tuple interface itself.
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
616 |
if len(key) == 2: |
617 |
prefix = key[:1] |
|
618 |
else: |
|
619 |
prefix = () |
|
620 |
parents = [prefix + (parent,) for parent in meta['parents']] |
|
621 |
vf_records.append((key, parents, meta['sha1'], d_func(text))) |
|
622 |
versionedfile.add_mpdiffs(vf_records) |
|
623 |
||
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
624 |
def _get_parent_inventory_texts(self, inventory_text_cache, |
625 |
inventory_cache, parent_ids): |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
626 |
cached_parent_texts = {} |
627 |
remaining_parent_ids = [] |
|
628 |
for parent_id in parent_ids: |
|
629 |
p_text = inventory_text_cache.get(parent_id, None) |
|
630 |
if p_text is None: |
|
631 |
remaining_parent_ids.append(parent_id) |
|
632 |
else: |
|
633 |
cached_parent_texts[parent_id] = p_text |
|
634 |
ghosts = () |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
635 |
# TODO: Use inventory_cache to grab inventories we already have in
|
636 |
# memory
|
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
637 |
if remaining_parent_ids: |
638 |
# first determine what keys are actually present in the local
|
|
639 |
# inventories object (don't use revisions as they haven't been
|
|
640 |
# installed yet.)
|
|
641 |
parent_keys = [(r,) for r in remaining_parent_ids] |
|
642 |
present_parent_map = self._repository.inventories.get_parent_map( |
|
643 |
parent_keys) |
|
644 |
present_parent_ids = [] |
|
645 |
ghosts = set() |
|
646 |
for p_id in remaining_parent_ids: |
|
647 |
if (p_id,) in present_parent_map: |
|
648 |
present_parent_ids.append(p_id) |
|
649 |
else: |
|
650 |
ghosts.add(p_id) |
|
651 |
to_string = self._source_serializer.write_inventory_to_string |
|
652 |
for parent_inv in self._repository.iter_inventories( |
|
653 |
present_parent_ids): |
|
654 |
p_text = to_string(parent_inv) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
655 |
inventory_cache[parent_inv.revision_id] = parent_inv |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
656 |
cached_parent_texts[parent_inv.revision_id] = p_text |
657 |
inventory_text_cache[parent_inv.revision_id] = p_text |
|
658 |
||
659 |
parent_texts = [cached_parent_texts[parent_id] |
|
660 |
for parent_id in parent_ids |
|
661 |
if parent_id not in ghosts] |
|
662 |
return parent_texts |
|
663 |
||
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
664 |
def _install_inventory_records(self, records): |
4543.2.3
by John Arbash Meinel
Change the name to test_merge_directive |
665 |
if (self._info['serializer'] == self._repository._serializer.format_num |
666 |
and self._repository._serializer.support_altered_by_hack): |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
667 |
return self._install_mp_records_keys(self._repository.inventories, |
668 |
records) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
669 |
# Use a 10MB text cache, since these are string xml inventories. Note
|
670 |
# that 10MB is fairly small for large projects (a single inventory can
|
|
671 |
# be >5MB). Another possibility is to cache 10-20 inventory texts
|
|
672 |
# instead
|
|
673 |
inventory_text_cache = lru_cache.LRUSizeCache(10*1024*1024) |
|
4543.2.21
by John Arbash Meinel
A few more tiny tweaks to comments, etc. |
674 |
# Also cache the in-memory representation. This allows us to create
|
675 |
# inventory deltas to apply rather than calling add_inventory from
|
|
676 |
# scratch each time.
|
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
677 |
inventory_cache = lru_cache.LRUCache(10) |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
678 |
pb = ui.ui_factory.nested_progress_bar() |
679 |
try: |
|
680 |
num_records = len(records) |
|
681 |
for idx, (key, metadata, bytes) in enumerate(records): |
|
682 |
pb.update('installing inventory', idx, num_records) |
|
683 |
revision_id = key[-1] |
|
684 |
parent_ids = metadata['parents'] |
|
685 |
# Note: This assumes the local ghosts are identical to the
|
|
686 |
# ghosts in the source, as the Bundle serialization
|
|
687 |
# format doesn't record ghosts.
|
|
688 |
p_texts = self._get_parent_inventory_texts(inventory_text_cache, |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
689 |
inventory_cache, |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
690 |
parent_ids) |
691 |
# Why does to_lines() take strings as the source, it seems that
|
|
692 |
# it would have to cast to a list of lines, which we get back
|
|
693 |
# as lines and then cast back to a string.
|
|
694 |
target_lines = multiparent.MultiParent.from_patch(bytes |
|
695 |
).to_lines(p_texts) |
|
696 |
inv_text = ''.join(target_lines) |
|
697 |
del target_lines |
|
698 |
sha1 = osutils.sha_string(inv_text) |
|
699 |
if sha1 != metadata['sha1']: |
|
700 |
raise errors.BadBundle("Can't convert to target format") |
|
701 |
# Add this to the cache so we don't have to extract it again.
|
|
702 |
inventory_text_cache[revision_id] = inv_text |
|
703 |
target_inv = self._source_serializer.read_inventory_from_string( |
|
704 |
inv_text) |
|
705 |
self._handle_root(target_inv, parent_ids) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
706 |
parent_inv = None |
707 |
if parent_ids: |
|
708 |
parent_inv = inventory_cache.get(parent_ids[0], None) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
709 |
try: |
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
710 |
if parent_inv is None: |
711 |
self._repository.add_inventory(revision_id, target_inv, |
|
712 |
parent_ids) |
|
713 |
else: |
|
714 |
delta = target_inv._make_delta(parent_inv) |
|
715 |
self._repository.add_inventory_by_delta(parent_ids[0], |
|
716 |
delta, revision_id, parent_ids) |
|
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
717 |
except errors.UnsupportedInventoryKind: |
718 |
raise errors.IncompatibleRevision(repr(self._repository)) |
|
4543.2.17
by John Arbash Meinel
Adding a parent inventory cache, and then using add_inventory_by_delta. |
719 |
inventory_cache[revision_id] = target_inv |
4543.2.16
by John Arbash Meinel
Adding an inventory text cache. |
720 |
finally: |
721 |
pb.finished() |
|
2520.4.99
by Aaron Bentley
Test conversion across models |
722 |
|
723 |
def _handle_root(self, target_inv, parent_ids): |
|
724 |
revision_id = target_inv.revision_id |
|
725 |
if self.update_root: |
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
726 |
text_key = (target_inv.root.file_id, revision_id) |
727 |
parent_keys = [(target_inv.root.file_id, parent) for |
|
728 |
parent in parent_ids] |
|
729 |
self._repository.texts.add_lines(text_key, parent_keys, []) |
|
2520.4.99
by Aaron Bentley
Test conversion across models |
730 |
elif not self._repository.supports_rich_root(): |
731 |
if target_inv.root.revision != revision_id: |
|
732 |
raise errors.IncompatibleRevision(repr(self._repository)) |
|
733 |
||
2520.4.59
by Aaron Bentley
Push metadata down the stack |
734 |
def _install_revision(self, revision_id, metadata, text): |
2520.4.14
by Aaron Bentley
Get most tests passing, use format header |
735 |
if self._repository.has_revision(revision_id): |
736 |
return
|
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
737 |
revision = self._source_serializer.read_revision_from_string(text) |
738 |
self._repository.add_revision(revision.revision_id, revision) |
|
2520.4.34
by Aaron Bentley
Add signature support |
739 |
|
2520.4.59
by Aaron Bentley
Push metadata down the stack |
740 |
def _install_signature(self, revision_id, metadata, text): |
2520.4.100
by Aaron Bentley
Fix repeat signature installs |
741 |
transaction = self._repository.get_transaction() |
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
742 |
if self._repository.has_signature_for_revision_id(revision_id): |
2520.4.100
by Aaron Bentley
Fix repeat signature installs |
743 |
return
|
3350.6.4
by Robert Collins
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores. |
744 |
self._repository.add_signature_text(revision_id, text) |