1
# Copyright (C) 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from cStringIO import StringIO
27
revision as _mod_revision,
31
from bzrlib.bundle import bundle_data, serializer
32
from bzrlib.util import bencode
35
class BundleWriter(object):
36
"""Writer for bundle-format files.
38
This serves roughly the same purpose as ContainerReader, but acts as a
41
Provides ways of writing the specific record types supported this bundle
45
def __init__(self, fileobj):
46
self._container = pack.ContainerWriter(self._write_encoded)
47
self._fileobj = fileobj
48
self._compressor = bz2.BZ2Compressor()
50
def _write_encoded(self, bytes):
51
"""Write bzip2-encoded bytes to the file"""
52
self._fileobj.write(self._compressor.compress(bytes))
55
"""Start writing the bundle"""
56
self._fileobj.write(serializer._get_bundle_header(
57
serializer.v4_string))
58
self._fileobj.write('#\n')
59
self._container.begin()
62
"""Finish writing the bundle"""
64
self._fileobj.write(self._compressor.flush())
66
def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
67
revision_id, file_id):
68
"""Add a record for a multi-parent diff
70
:mp_bytes: A multi-parent diff, as a bytestring
71
:sha1: The sha1 hash of the fulltext
72
:parents: a list of revision-ids of the parents
73
:repo_kind: The kind of object in the repository. May be 'file' or
75
:revision_id: The revision id of the mpdiff being added.
76
:file_id: The file-id of the file, or None for inventories.
78
metadata = {'parents': parents,
79
'storage_kind': 'mpdiff',
81
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
83
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
84
"""Add a record for a fulltext
86
:bytes: The fulltext, as a bytestring
87
:parents: a list of revision-ids of the parents
88
:repo_kind: The kind of object in the repository. May be 'revision' or
90
:revision_id: The revision id of the fulltext being added.
92
metadata = {'parents': parents,
93
'storage_kind': 'mpdiff'}
94
self._add_record(bytes, {'parents': parents,
95
'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
97
def add_info_record(self, **kwargs):
98
"""Add an info record to the bundle
100
Any parameters may be supplied, except 'self' and 'storage_kind'.
101
Values must be lists, strings, integers, dicts, or a combination.
103
kwargs['storage_kind'] = 'header'
104
self._add_record(None, kwargs, 'info', None, None)
107
def encode_name(content_kind, revision_id, file_id=None):
108
"""Encode semantic ids as a container name"""
109
assert content_kind in ('revision', 'file', 'inventory', 'signature',
112
if content_kind == 'file':
113
assert file_id is not None
115
assert file_id is None
116
if content_kind == 'info':
117
assert revision_id is None
119
assert revision_id is not None
120
names = [n.replace('/', '//') for n in
121
(content_kind, revision_id, file_id) if n is not None]
122
return '/'.join(names)
124
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
125
"""Add a bundle record to the container.
127
Most bundle records are recorded as header/body pairs, with the
128
body being nameless. Records with storage_kind 'header' have no
131
name = self.encode_name(repo_kind, revision_id, file_id)
132
encoded_metadata = bencode.bencode(metadata)
133
self._container.add_bytes_record(encoded_metadata, [name])
134
if metadata['storage_kind'] != 'header':
135
self._container.add_bytes_record(bytes, [])
138
class BundleReader(object):
139
"""Reader for bundle-format files.
141
This serves roughly the same purpose as ContainerReader, but acts as a
142
layer on top of it, providing metadata, a semantic name, and a record
146
def __init__(self, fileobj):
147
line = fileobj.readline()
150
self.patch_lines = []
151
self._container = pack.ContainerReader(
152
iterablefile.IterableFile(self.iter_decode(fileobj)))
155
def iter_decode(fileobj):
156
"""Iterate through decoded fragments of the file"""
157
decompressor = bz2.BZ2Decompressor()
159
yield decompressor.decompress(line)
162
def decode_name(name):
163
"""Decode a name from its container form into a semantic form
165
:retval: content_kind, revision_id, file_id
167
segments = re.split('//?', name)
169
for segment in segments:
176
content_kind = name[0]
180
revision_id = names[1]
183
return content_kind, revision_id, file_id
185
def iter_records(self):
186
"""Iterate through bundle records
188
:return: a generator of (bytes, metadata, content_kind, revision_id,
191
iterator = self._container.iter_records()
192
for (name,), meta_bytes in iterator:
193
metadata = bencode.bdecode(meta_bytes(None))
194
if metadata['storage_kind'] == 'header':
197
_unused, bytes = iterator.next()
199
yield (bytes, metadata) + self.decode_name(name)
202
class BundleSerializerV4(serializer.BundleSerializer):
203
"""Implement the high-level bundle interface"""
205
def write(self, repository, revision_ids, forced_bases, fileobj):
206
"""Write a bundle to a file-like object
208
For backwards-compatibility only
210
write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
211
forced_bases, fileobj)
212
return write_op.do_write()
214
def write_bundle(self, repository, target, base, fileobj):
215
"""Write a bundle to a file object
217
:param repository: The repository to retrieve revision data from
218
:param target: The head revision to include ancestors of
219
:param base: The ancestor of the target to stop including acestors
221
:param fileobj: The file-like object to write to
223
write_op = BundleWriteOperation(base, target, repository, fileobj)
224
return write_op.do_write()
226
def read(self, file):
227
"""return a reader object for a given file"""
228
bundle = BundleInfoV4(file, self)
232
def get_source_serializer(info):
233
"""Retrieve the serializer for a given info object"""
234
return xml_serializer.format_registry.get(info['serializer'])
237
class BundleWriteOperation(object):
238
"""Perform the operation of writing revisions to a bundle"""
241
def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
242
"""Create a BundleWriteOperation from old-style arguments"""
243
base, target = cls.get_base_target(revision_ids, forced_bases,
245
return BundleWriteOperation(base, target, repository, fileobj,
248
def __init__(self, base, target, repository, fileobj, revision_ids=None):
251
self.repository = repository
252
bundle = BundleWriter(fileobj)
254
self.base_ancestry = set(repository.get_ancestry(base,
256
if revision_ids is not None:
257
self.revision_ids = revision_ids
259
revision_ids = set(repository.get_ancestry(target,
261
self.revision_ids = revision_ids.difference(self.base_ancestry)
264
"""Write all data to the bundle"""
268
self.write_revisions()
270
return self.revision_ids
272
def write_info(self):
273
"""Write format info"""
274
serializer_format = self.repository.get_serializer_format()
275
supports_rich_root = {True: 1, False: 0}[
276
self.repository.supports_rich_root()]
277
self.bundle.add_info_record(serializer=serializer_format,
278
supports_rich_root=supports_rich_root)
280
def iter_file_revisions(self):
281
"""Iterate through all relevant revisions of all files.
283
This is the correct implementation, but is not compatible with bzr.dev,
284
because certain old revisions were not converted correctly, and have
285
the wrong "revision" marker in inventories.
287
transaction = self.repository.get_transaction()
288
altered = self.repository.fileids_altered_by_revision_ids(
290
for file_id, file_revision_ids in altered.iteritems():
291
vf = self.repository.weave_store.get_weave(file_id, transaction)
292
yield vf, file_id, file_revision_ids
294
def iter_file_revisions_aggressive(self):
295
"""Iterate through all relevant revisions of all files.
297
This uses the standard iter_file_revisions to determine what revisions
298
are referred to by inventories, but then uses the versionedfile to
299
determine what the build-dependencies of each required revision.
301
All build dependencies which are not ancestors of the base revision
304
for vf, file_id, file_revision_ids in self.iter_file_revisions():
305
new_revision_ids = set()
306
pending = list(file_revision_ids)
307
while len(pending) > 0:
308
revision_id = pending.pop()
309
if revision_id in new_revision_ids:
311
if revision_id in self.base_ancestry:
313
new_revision_ids.add(revision_id)
314
pending.extend(vf.get_parents(revision_id))
315
yield vf, file_id, new_revision_ids
317
def write_files(self):
318
"""Write bundle records for all revisions of all files"""
319
for vf, file_id, revision_ids in self.iter_file_revisions_aggressive():
320
self.add_mp_records('file', file_id, vf, revision_ids)
322
def write_revisions(self):
323
"""Write bundle records for all revisions and signatures"""
324
inv_vf = self.repository.get_inventory_weave()
325
revision_order = list(multiparent.topo_iter(inv_vf, self.revision_ids))
326
if self.target is not None and self.target in self.revision_ids:
327
revision_order.remove(self.target)
328
revision_order.append(self.target)
329
self.add_mp_records('inventory', None, inv_vf, revision_order)
330
parents_list = self.repository.get_parents(revision_order)
331
for parents, revision_id in zip(parents_list, revision_order):
332
revision_text = self.repository.get_revision_xml(revision_id)
333
self.bundle.add_fulltext_record(revision_text, parents,
334
'revision', revision_id)
336
self.bundle.add_fulltext_record(
337
self.repository.get_signature_text(
338
revision_id), parents, 'signature', revision_id)
339
except errors.NoSuchRevision:
343
def get_base_target(revision_ids, forced_bases, repository):
344
"""Determine the base and target from old-style revision ids"""
345
if len(revision_ids) == 0:
347
target = revision_ids[0]
348
base = forced_bases.get(target)
350
parents = repository.get_revision(target).parent_ids
351
if len(parents) == 0:
352
base = _mod_revision.NULL_REVISION
357
def add_mp_records(self, repo_kind, file_id, vf, revision_ids):
358
"""Add multi-parent diff records to a bundle"""
359
revision_ids = list(multiparent.topo_iter(vf, revision_ids))
360
mpdiffs = vf.make_mpdiffs(revision_ids)
361
sha1s = vf.get_sha1s(revision_ids)
362
for mpdiff, revision_id, sha1, in zip(mpdiffs, revision_ids, sha1s):
363
parents = vf.get_parents(revision_id)
364
text = ''.join(mpdiff.to_patch())
365
self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
366
revision_id, file_id)
369
class BundleInfoV4(object):
371
"""Provide (most of) the BundleInfo interface"""
372
def __init__(self, fileobj, serializer):
373
self._fileobj = fileobj
374
self._serializer = serializer
375
self.__real_revisions = None
376
self.__revisions = None
378
def install(self, repository):
379
return self.install_revisions(repository)
381
def install_revisions(self, repository):
382
"""Install this bundle's revisions into the specified repository"""
383
repository.lock_write()
385
ri = RevisionInstaller(self.get_bundle_reader(),
386
self._serializer, repository)
391
def get_merge_request(self, target_repo):
392
"""Provide data for performing a merge
394
Returns suggested base, suggested target, and patch verification status
396
return None, self.target, 'inapplicable'
398
def get_bundle_reader(self):
399
self._fileobj.seek(0)
400
return BundleReader(self._fileobj)
402
def _get_real_revisions(self):
403
if self.__real_revisions is None:
404
self.__real_revisions = []
405
bundle_reader = self.get_bundle_reader()
406
for bytes, metadata, repo_kind, revision_id, file_id in \
407
bundle_reader.iter_records():
408
if repo_kind == 'info':
410
self._serializer.get_source_serializer(metadata)
411
if repo_kind == 'revision':
412
rev = serializer.read_revision_from_string(bytes)
413
self.__real_revisions.append(rev)
414
return self.__real_revisions
415
real_revisions = property(_get_real_revisions)
417
def _get_revisions(self):
418
if self.__revisions is None:
419
self.__revisions = []
420
for revision in self.real_revisions:
421
self.__revisions.append(
422
bundle_data.RevisionInfo.from_revision(revision))
423
return self.__revisions
425
revisions = property(_get_revisions)
427
def _get_target(self):
428
return self.revisions[-1].revision_id
430
target = property(_get_target)
433
class RevisionInstaller(object):
434
"""Installs revisions into a repository"""
436
def __init__(self, container, serializer, repository):
437
self._container = container
438
self._serializer = serializer
439
self._repository = repository
443
"""Perform the installation"""
445
current_versionedfile = None
446
pending_file_records = []
448
target_revision = None
449
for bytes, metadata, repo_kind, revision_id, file_id in\
450
self._container.iter_records():
451
if repo_kind == 'info':
452
assert self._info is None
453
self._handle_info(metadata)
454
if repo_kind != 'file':
455
self._install_mp_records(current_versionedfile,
456
pending_file_records)
458
current_versionedfile = None
459
pending_file_records = []
460
if repo_kind == 'inventory':
461
self._install_inventory(revision_id, metadata, bytes)
462
if repo_kind == 'revision':
463
target_revision = revision_id
464
self._install_revision(revision_id, metadata, bytes)
465
if repo_kind == 'signature':
466
self._install_signature(revision_id, metadata, bytes)
467
if repo_kind == 'file':
468
if file_id != current_file:
469
self._install_mp_records(current_versionedfile,
470
pending_file_records)
471
current_file = file_id
472
current_versionedfile = \
473
self._repository.weave_store.get_weave_or_empty(
474
file_id, self._repository.get_transaction())
475
pending_file_records = []
476
if revision_id in current_versionedfile:
478
pending_file_records.append((revision_id, metadata, bytes))
479
self._install_mp_records(current_versionedfile, pending_file_records)
480
return target_revision
482
def _handle_info(self, info):
483
"""Extract data from an info record"""
485
self._source_serializer = self._serializer.get_source_serializer(info)
486
if (info['supports_rich_root'] == 0 and
487
self._repository.supports_rich_root()):
488
self.update_root = True
490
self.update_root = False
492
def _install_mp_records(self, versionedfile, records):
493
if len(records) == 0:
495
d_func = multiparent.MultiParent.from_patch
496
vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
497
records if r not in versionedfile]
498
versionedfile.add_mpdiffs(vf_records)
500
def _install_inventory(self, revision_id, metadata, text):
501
vf = self._repository.get_inventory_weave()
502
if revision_id in vf:
504
parent_ids = metadata['parents']
505
if self._info['serializer'] == self._repository._serializer.format_num:
506
return self._install_mp_records(vf, [(revision_id, metadata,
508
parents = [self._repository.get_inventory(p)
510
parent_texts = [self._source_serializer.write_inventory_to_string(p)
512
target_lines = multiparent.MultiParent.from_patch(text).to_lines(
514
sha1 = osutils.sha_strings(target_lines)
515
if sha1 != metadata['sha1']:
516
raise BadBundle("Can't convert to target format")
517
target_inv = self._source_serializer.read_inventory_from_string(
518
''.join(target_lines))
519
self._handle_root(target_inv, parent_ids)
521
self._repository.add_inventory(revision_id, target_inv, parent_ids)
522
except errors.UnsupportedInventoryKind:
523
raise errors.IncompatibleRevision(repr(self._repository))
525
def _handle_root(self, target_inv, parent_ids):
526
revision_id = target_inv.revision_id
528
target_inv.root.revision = revision_id
529
store = self._repository.weave_store
530
transaction = self._repository.get_transaction()
531
vf = store.get_weave_or_empty(target_inv.root.file_id, transaction)
532
vf.add_lines(revision_id, parent_ids, [])
533
elif not self._repository.supports_rich_root():
534
if target_inv.root.revision != revision_id:
535
raise errors.IncompatibleRevision(repr(self._repository))
538
def _install_revision(self, revision_id, metadata, text):
539
if self._repository.has_revision(revision_id):
541
self._repository._add_revision_text(revision_id, text)
543
def _install_signature(self, revision_id, metadata, text):
544
transaction = self._repository.get_transaction()
545
if self._repository._revision_store.has_signature(revision_id,
548
self._repository._revision_store.add_revision_signature_text(
549
revision_id, text, transaction)
added
removed
bzrlib/bundle/serializer/v4.py
