1
# Copyright (C) 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from cStringIO import StringIO
28
revision as _mod_revision,
32
from bzrlib.bundle import bundle_data, serializer
33
from bzrlib.util import bencode
36
class BundleWriter(object):
37
"""Writer for bundle-format files.
39
This serves roughly the same purpose as ContainerReader, but acts as a
42
Provides ways of writing the specific record types supported this bundle
46
def __init__(self, fileobj):
47
self._container = pack.ContainerWriter(self._write_encoded)
48
self._fileobj = fileobj
49
self._compressor = bz2.BZ2Compressor()
51
def _write_encoded(self, bytes):
52
"""Write bzip2-encoded bytes to the file"""
53
self._fileobj.write(self._compressor.compress(bytes))
56
"""Start writing the bundle"""
57
self._fileobj.write(serializer._get_bundle_header(
58
serializer.v4_string))
59
self._fileobj.write('#\n')
60
self._container.begin()
63
"""Finish writing the bundle"""
65
self._fileobj.write(self._compressor.flush())
67
def add_multiparent_record(self, mp_bytes, sha1, parents, repo_kind,
68
revision_id, file_id):
69
"""Add a record for a multi-parent diff
71
:mp_bytes: A multi-parent diff, as a bytestring
72
:sha1: The sha1 hash of the fulltext
73
:parents: a list of revision-ids of the parents
74
:repo_kind: The kind of object in the repository. May be 'file' or
76
:revision_id: The revision id of the mpdiff being added.
77
:file_id: The file-id of the file, or None for inventories.
79
metadata = {'parents': parents,
80
'storage_kind': 'mpdiff',
82
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
84
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
85
"""Add a record for a fulltext
87
:bytes: The fulltext, as a bytestring
88
:parents: a list of revision-ids of the parents
89
:repo_kind: The kind of object in the repository. May be 'revision' or
91
:revision_id: The revision id of the fulltext being added.
93
metadata = {'parents': parents,
94
'storage_kind': 'mpdiff'}
95
self._add_record(bytes, {'parents': parents,
96
'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
98
def add_info_record(self, **kwargs):
99
"""Add an info record to the bundle
101
Any parameters may be supplied, except 'self' and 'storage_kind'.
102
Values must be lists, strings, integers, dicts, or a combination.
104
kwargs['storage_kind'] = 'header'
105
self._add_record(None, kwargs, 'info', None, None)
108
def encode_name(content_kind, revision_id, file_id=None):
109
"""Encode semantic ids as a container name"""
110
assert content_kind in ('revision', 'file', 'inventory', 'signature',
113
if content_kind == 'file':
114
assert file_id is not None
116
assert file_id is None
117
if content_kind == 'info':
118
assert revision_id is None
120
assert revision_id is not None
121
names = [n.replace('/', '//') for n in
122
(content_kind, revision_id, file_id) if n is not None]
123
return '/'.join(names)
125
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
126
"""Add a bundle record to the container.
128
Most bundle records are recorded as header/body pairs, with the
129
body being nameless. Records with storage_kind 'header' have no
132
name = self.encode_name(repo_kind, revision_id, file_id)
133
encoded_metadata = bencode.bencode(metadata)
134
self._container.add_bytes_record(encoded_metadata, [name])
135
if metadata['storage_kind'] != 'header':
136
self._container.add_bytes_record(bytes, [])
139
class BundleReader(object):
140
"""Reader for bundle-format files.
142
This serves roughly the same purpose as ContainerReader, but acts as a
143
layer on top of it, providing metadata, a semantic name, and a record
147
def __init__(self, fileobj):
148
line = fileobj.readline()
151
self.patch_lines = []
152
self._container = pack.ContainerReader(
153
iterablefile.IterableFile(self.iter_decode(fileobj)))
156
def iter_decode(fileobj):
157
"""Iterate through decoded fragments of the file"""
158
decompressor = bz2.BZ2Decompressor()
160
yield decompressor.decompress(line)
163
def decode_name(name):
164
"""Decode a name from its container form into a semantic form
166
:retval: content_kind, revision_id, file_id
168
segments = re.split('(//?)', name)
170
for segment in segments:
177
content_kind = names[0]
181
revision_id = names[1]
184
return content_kind, revision_id, file_id
186
def iter_records(self):
187
"""Iterate through bundle records
189
:return: a generator of (bytes, metadata, content_kind, revision_id,
192
iterator = self._container.iter_records()
193
for names, meta_bytes in iterator:
195
raise errors.BadBundle('Record has %d names instead of 1'
197
metadata = bencode.bdecode(meta_bytes(None))
198
if metadata['storage_kind'] == 'header':
201
_unused, bytes = iterator.next()
203
yield (bytes, metadata) + self.decode_name(names[0])
206
class BundleSerializerV4(serializer.BundleSerializer):
207
"""Implement the high-level bundle interface"""
209
def write(self, repository, revision_ids, forced_bases, fileobj):
210
"""Write a bundle to a file-like object
212
For backwards-compatibility only
214
write_op = BundleWriteOperation.from_old_args(repository, revision_ids,
215
forced_bases, fileobj)
216
return write_op.do_write()
218
def write_bundle(self, repository, target, base, fileobj):
219
"""Write a bundle to a file object
221
:param repository: The repository to retrieve revision data from
222
:param target: The head revision to include ancestors of
223
:param base: The ancestor of the target to stop including acestors
225
:param fileobj: The file-like object to write to
227
write_op = BundleWriteOperation(base, target, repository, fileobj)
228
return write_op.do_write()
230
def read(self, file):
231
"""return a reader object for a given file"""
232
bundle = BundleInfoV4(file, self)
236
def get_source_serializer(info):
237
"""Retrieve the serializer for a given info object"""
238
return xml_serializer.format_registry.get(info['serializer'])
241
class BundleWriteOperation(object):
242
"""Perform the operation of writing revisions to a bundle"""
245
def from_old_args(cls, repository, revision_ids, forced_bases, fileobj):
246
"""Create a BundleWriteOperation from old-style arguments"""
247
base, target = cls.get_base_target(revision_ids, forced_bases,
249
return BundleWriteOperation(base, target, repository, fileobj,
252
def __init__(self, base, target, repository, fileobj, revision_ids=None):
255
self.repository = repository
256
bundle = BundleWriter(fileobj)
258
self.base_ancestry = set(repository.get_ancestry(base,
260
if revision_ids is not None:
261
self.revision_ids = revision_ids
263
revision_ids = set(repository.get_ancestry(target,
265
self.revision_ids = revision_ids.difference(self.base_ancestry)
268
"""Write all data to the bundle"""
272
self.write_revisions()
274
return self.revision_ids
276
def write_info(self):
277
"""Write format info"""
278
serializer_format = self.repository.get_serializer_format()
279
supports_rich_root = {True: 1, False: 0}[
280
self.repository.supports_rich_root()]
281
self.bundle.add_info_record(serializer=serializer_format,
282
supports_rich_root=supports_rich_root)
284
def iter_file_revisions(self):
285
"""Iterate through all relevant revisions of all files.
287
This is the correct implementation, but is not compatible with bzr.dev,
288
because certain old revisions were not converted correctly, and have
289
the wrong "revision" marker in inventories.
291
transaction = self.repository.get_transaction()
292
altered = self.repository.fileids_altered_by_revision_ids(
294
for file_id, file_revision_ids in altered.iteritems():
295
vf = self.repository.weave_store.get_weave(file_id, transaction)
296
yield vf, file_id, file_revision_ids
298
def iter_file_revisions_aggressive(self):
299
"""Iterate through all relevant revisions of all files.
301
This uses the standard iter_file_revisions to determine what revisions
302
are referred to by inventories, but then uses the versionedfile to
303
determine what the build-dependencies of each required revision.
305
All build dependencies which are not ancestors of the base revision
308
for vf, file_id, file_revision_ids in self.iter_file_revisions():
309
new_revision_ids = set()
310
pending = list(file_revision_ids)
311
while len(pending) > 0:
312
revision_id = pending.pop()
313
if revision_id in new_revision_ids:
315
if revision_id in self.base_ancestry:
317
new_revision_ids.add(revision_id)
318
pending.extend(vf.get_parents(revision_id))
319
yield vf, file_id, new_revision_ids
321
def write_files(self):
322
"""Write bundle records for all revisions of all files"""
323
for vf, file_id, revision_ids in self.iter_file_revisions_aggressive():
324
self.add_mp_records('file', file_id, vf, revision_ids)
326
def write_revisions(self):
327
"""Write bundle records for all revisions and signatures"""
328
inv_vf = self.repository.get_inventory_weave()
329
revision_order = list(multiparent.topo_iter(inv_vf, self.revision_ids))
330
if self.target is not None and self.target in self.revision_ids:
331
revision_order.remove(self.target)
332
revision_order.append(self.target)
333
self.add_mp_records('inventory', None, inv_vf, revision_order)
334
parents_list = self.repository.get_parents(revision_order)
335
for parents, revision_id in zip(parents_list, revision_order):
336
revision_text = self.repository.get_revision_xml(revision_id)
337
self.bundle.add_fulltext_record(revision_text, parents,
338
'revision', revision_id)
340
self.bundle.add_fulltext_record(
341
self.repository.get_signature_text(
342
revision_id), parents, 'signature', revision_id)
343
except errors.NoSuchRevision:
347
def get_base_target(revision_ids, forced_bases, repository):
348
"""Determine the base and target from old-style revision ids"""
349
if len(revision_ids) == 0:
351
target = revision_ids[0]
352
base = forced_bases.get(target)
354
parents = repository.get_revision(target).parent_ids
355
if len(parents) == 0:
356
base = _mod_revision.NULL_REVISION
361
def add_mp_records(self, repo_kind, file_id, vf, revision_ids):
362
"""Add multi-parent diff records to a bundle"""
363
revision_ids = list(multiparent.topo_iter(vf, revision_ids))
364
mpdiffs = vf.make_mpdiffs(revision_ids)
365
sha1s = vf.get_sha1s(revision_ids)
366
for mpdiff, revision_id, sha1, in zip(mpdiffs, revision_ids, sha1s):
367
parents = vf.get_parents(revision_id)
368
text = ''.join(mpdiff.to_patch())
369
self.bundle.add_multiparent_record(text, sha1, parents, repo_kind,
370
revision_id, file_id)
373
class BundleInfoV4(object):
375
"""Provide (most of) the BundleInfo interface"""
376
def __init__(self, fileobj, serializer):
377
self._fileobj = fileobj
378
self._serializer = serializer
379
self.__real_revisions = None
380
self.__revisions = None
382
def install(self, repository):
383
return self.install_revisions(repository)
385
def install_revisions(self, repository):
386
"""Install this bundle's revisions into the specified repository"""
387
repository.lock_write()
389
ri = RevisionInstaller(self.get_bundle_reader(),
390
self._serializer, repository)
395
def get_merge_request(self, target_repo):
396
"""Provide data for performing a merge
398
Returns suggested base, suggested target, and patch verification status
400
return None, self.target, 'inapplicable'
402
def get_bundle_reader(self):
403
self._fileobj.seek(0)
404
return BundleReader(self._fileobj)
406
def _get_real_revisions(self):
407
if self.__real_revisions is None:
408
self.__real_revisions = []
409
bundle_reader = self.get_bundle_reader()
410
for bytes, metadata, repo_kind, revision_id, file_id in \
411
bundle_reader.iter_records():
412
if repo_kind == 'info':
414
self._serializer.get_source_serializer(metadata)
415
if repo_kind == 'revision':
416
rev = serializer.read_revision_from_string(bytes)
417
self.__real_revisions.append(rev)
418
return self.__real_revisions
419
real_revisions = property(_get_real_revisions)
421
def _get_revisions(self):
422
if self.__revisions is None:
423
self.__revisions = []
424
for revision in self.real_revisions:
425
self.__revisions.append(
426
bundle_data.RevisionInfo.from_revision(revision))
427
return self.__revisions
429
revisions = property(_get_revisions)
431
def _get_target(self):
432
return self.revisions[-1].revision_id
434
target = property(_get_target)
437
class RevisionInstaller(object):
438
"""Installs revisions into a repository"""
440
def __init__(self, container, serializer, repository):
441
self._container = container
442
self._serializer = serializer
443
self._repository = repository
447
"""Perform the installation"""
449
current_versionedfile = None
450
pending_file_records = []
452
pending_inventory_records = []
454
target_revision = None
455
for bytes, metadata, repo_kind, revision_id, file_id in\
456
self._container.iter_records():
457
if repo_kind == 'info':
458
assert self._info is None
459
self._handle_info(metadata)
460
if (repo_kind, file_id) != ('file', current_file) and\
461
len(pending_file_records) > 0:
462
self._install_mp_records(current_versionedfile,
463
pending_file_records)
465
current_versionedfile = None
466
pending_file_records = []
467
if len(pending_inventory_records) > 0 and repo_kind != 'inventory':
468
self._install_inventory_records(inventory_vf,
469
pending_inventory_records)
470
pending_inventory_records = []
471
if repo_kind == 'inventory':
472
if inventory_vf is None:
473
inventory_vf = self._repository.get_inventory_weave()
474
if revision_id not in inventory_vf:
475
pending_inventory_records.append((revision_id, metadata,
477
if repo_kind == 'revision':
478
target_revision = revision_id
479
self._install_revision(revision_id, metadata, bytes)
480
if repo_kind == 'signature':
481
self._install_signature(revision_id, metadata, bytes)
482
if repo_kind == 'file':
483
current_file = file_id
484
if current_versionedfile is None:
485
current_versionedfile = \
486
self._repository.weave_store.get_weave_or_empty(
487
file_id, self._repository.get_transaction())
488
pending_file_records = []
489
if revision_id in current_versionedfile:
491
pending_file_records.append((revision_id, metadata, bytes))
492
self._install_mp_records(current_versionedfile, pending_file_records)
493
return target_revision
495
def _handle_info(self, info):
496
"""Extract data from an info record"""
498
self._source_serializer = self._serializer.get_source_serializer(info)
499
if (info['supports_rich_root'] == 0 and
500
self._repository.supports_rich_root()):
501
self.update_root = True
503
self.update_root = False
505
def _install_mp_records(self, versionedfile, records):
506
if len(records) == 0:
508
d_func = multiparent.MultiParent.from_patch
509
vf_records = [(r, m['parents'], m['sha1'], d_func(t)) for r, m, t in
510
records if r not in versionedfile]
511
versionedfile.add_mpdiffs(vf_records)
513
def _install_inventory_records(self, vf, records):
514
if self._info['serializer'] == self._repository._serializer.format_num:
515
return self._install_mp_records(vf, records)
516
for revision_id, metadata, bytes in records:
517
parent_ids = metadata['parents']
518
parents = [self._repository.get_inventory(p)
520
p_texts = [self._source_serializer.write_inventory_to_string(p)
522
target_lines = multiparent.MultiParent.from_patch(bytes).to_lines(
524
sha1 = osutils.sha_strings(target_lines)
525
if sha1 != metadata['sha1']:
526
raise errors.BadBundle("Can't convert to target format")
527
target_inv = self._source_serializer.read_inventory_from_string(
528
''.join(target_lines))
529
self._handle_root(target_inv, parent_ids)
531
self._repository.add_inventory(revision_id, target_inv,
533
except errors.UnsupportedInventoryKind:
534
raise errors.IncompatibleRevision(repr(self._repository))
536
def _handle_root(self, target_inv, parent_ids):
537
revision_id = target_inv.revision_id
539
target_inv.root.revision = revision_id
540
store = self._repository.weave_store
541
transaction = self._repository.get_transaction()
542
vf = store.get_weave_or_empty(target_inv.root.file_id, transaction)
543
vf.add_lines(revision_id, parent_ids, [])
544
elif not self._repository.supports_rich_root():
545
if target_inv.root.revision != revision_id:
546
raise errors.IncompatibleRevision(repr(self._repository))
549
def _install_revision(self, revision_id, metadata, text):
550
if self._repository.has_revision(revision_id):
552
self._repository._add_revision_text(revision_id, text)
554
def _install_signature(self, revision_id, metadata, text):
555
transaction = self._repository.get_transaction()
556
if self._repository._revision_store.has_signature(revision_id,
559
self._repository._revision_store.add_revision_signature_text(
560
revision_id, text, transaction)
added
removed
bzrlib/bundle/serializer/v4.py
