14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
from cStringIO import StringIO
17
from __future__ import absolute_import
29
29
revision as _mod_revision,
34
from bzrlib.bundle import bundle_data, serializer as bundle_serializer
35
from bzrlib import bencode
36
versionedfile as _mod_versionedfile,
38
from ...bundle import bundle_data, serializer as bundle_serializer
39
from ...i18n import ngettext
40
from ...sixish import (
46
class _MPDiffInventoryGenerator(_mod_versionedfile._MPDiffGenerator):
47
"""Generate Inventory diffs serialized inventories."""
49
def __init__(self, repo, inventory_keys):
50
super(_MPDiffInventoryGenerator, self).__init__(repo.inventories,
56
"""Compute the diffs one at a time."""
57
# This is instead of compute_diffs() since we guarantee our ordering of
58
# inventories, we don't have to do any buffering
59
self._find_needed_keys()
60
# We actually use a slightly different ordering. We grab all of the
61
# parents first, and then grab the ordered requests.
62
needed_ids = [k[-1] for k in self.present_parents]
63
needed_ids.extend([k[-1] for k in self.ordered_keys])
64
inv_to_str = self.repo._serializer.write_inventory_to_string
65
for inv in self.repo.iter_inventories(needed_ids):
66
revision_id = inv.revision_id
68
if key in self.present_parents:
69
# Not a key we will transmit, which is a shame, since because
70
# of that bundles don't work with stacked branches
73
parent_ids = [k[-1] for k in self.parent_map[key]]
74
as_bytes = inv_to_str(inv)
75
self._process_one_record(key, (as_bytes,))
76
if parent_ids is None:
78
diff = self.diffs.pop(key)
79
sha1 = osutils.sha_string(as_bytes)
80
yield revision_id, parent_ids, sha1, diff
38
83
class BundleWriter(object):
58
103
"""Start writing the bundle"""
59
self._fileobj.write(bundle_serializer._get_bundle_header(
60
bundle_serializer.v4_string))
61
self._fileobj.write('#\n')
104
self._fileobj.write(bundle_serializer._get_bundle_header('4'))
105
self._fileobj.write(b'#\n')
62
106
self._container.begin()
78
122
:revision_id: The revision id of the mpdiff being added.
79
123
:file_id: The file-id of the file, or None for inventories.
81
metadata = {'parents': parents,
82
'storage_kind': 'mpdiff',
125
metadata = {b'parents': parents,
126
b'storage_kind': b'mpdiff',
84
128
self._add_record(mp_bytes, metadata, repo_kind, revision_id, file_id)
86
130
def add_fulltext_record(self, bytes, parents, repo_kind, revision_id):
93
137
:revision_id: The revision id of the fulltext being added.
95
metadata = {'parents': parents,
96
'storage_kind': 'mpdiff'}
97
self._add_record(bytes, {'parents': parents,
98
'storage_kind': 'fulltext'}, repo_kind, revision_id, None)
139
metadata = {b'parents': parents,
140
b'storage_kind': b'mpdiff'}
141
self._add_record(bytes, {b'parents': parents,
142
b'storage_kind': b'fulltext'}, repo_kind, revision_id, None)
100
def add_info_record(self, **kwargs):
144
def add_info_record(self, kwargs):
101
145
"""Add an info record to the bundle
103
147
Any parameters may be supplied, except 'self' and 'storage_kind'.
104
148
Values must be lists, strings, integers, dicts, or a combination.
106
kwargs['storage_kind'] = 'header'
150
kwargs[b'storage_kind'] = b'header'
107
151
self._add_record(None, kwargs, 'info', None, None)
123
167
raise AssertionError()
124
168
elif revision_id is None:
125
169
raise AssertionError()
126
names = [n.replace('/', '//') for n in
127
(content_kind, revision_id, file_id) if n is not None]
128
return '/'.join(names)
170
names = [n.replace(b'/', b'//') for n in
171
(content_kind.encode('ascii'), revision_id, file_id) if n is not None]
172
return b'/'.join(names)
130
174
def _add_record(self, bytes, metadata, repo_kind, revision_id, file_id):
131
175
"""Add a bundle record to the container.
137
181
name = self.encode_name(repo_kind, revision_id, file_id)
138
182
encoded_metadata = bencode.bencode(metadata)
139
183
self._container.add_bytes_record(encoded_metadata, [(name, )])
140
if metadata['storage_kind'] != 'header':
184
if metadata[b'storage_kind'] != b'header':
141
185
self._container.add_bytes_record(bytes, [])
213
257
raise errors.BadBundle('Record has %d names instead of 1'
215
259
metadata = bencode.bdecode(bytes)
216
if metadata['storage_kind'] == 'header':
260
if metadata[b'storage_kind'] == b'header':
219
_unused, bytes = iterator.next()
263
_unused, bytes = next(iterator)
220
264
yield (bytes, metadata) + self.decode_name(names[0][0])
281
325
parents = graph.get_parent_map(revision_ids)
282
326
self.revision_ids = [r for r in revision_ids if r in parents]
283
self.revision_keys = set([(revid,) for revid in self.revision_ids])
327
self.revision_keys = {(revid,) for revid in self.revision_ids}
285
329
def do_write(self):
286
330
"""Write all data to the bundle"""
287
trace.note('Bundling %d revision(s).', len(self.revision_ids))
288
self.repository.lock_read()
331
trace.note(ngettext('Bundling %d revision.', 'Bundling %d revisions.',
332
len(self.revision_ids)), len(self.revision_ids))
333
with self.repository.lock_read():
290
334
self.bundle.begin()
291
335
self.write_info()
292
336
self.write_files()
293
337
self.write_revisions()
294
338
self.bundle.end()
296
self.repository.unlock()
297
339
return self.revision_ids
299
341
def write_info(self):
301
343
serializer_format = self.repository.get_serializer_format()
302
344
supports_rich_root = {True: 1, False: 0}[
303
345
self.repository.supports_rich_root()]
304
self.bundle.add_info_record(serializer=serializer_format,
305
supports_rich_root=supports_rich_root)
346
self.bundle.add_info_record({b'serializer': serializer_format,
347
b'supports_rich_root': supports_rich_root})
307
349
def write_files(self):
308
350
"""Write bundle records for all revisions of all files"""
310
352
altered_fileids = self.repository.fileids_altered_by_revision_ids(
311
353
self.revision_ids)
312
for file_id, revision_ids in altered_fileids.iteritems():
354
for file_id, revision_ids in viewitems(altered_fileids):
313
355
for revision_id in revision_ids:
314
356
text_keys.append((file_id, revision_id))
315
357
self._add_mp_records_keys('file', self.repository.texts, text_keys)
350
392
inventory_key_order = [(r,) for r in revision_order]
351
parent_map = self.repository.inventories.get_parent_map(
353
missing_keys = set(inventory_key_order).difference(parent_map)
355
raise errors.RevisionNotPresent(list(missing_keys)[0],
356
self.repository.inventories)
357
inv_to_str = self.repository._serializer.write_inventory_to_string
358
# Make sure that we grab the parent texts first
360
map(just_parents.update, parent_map.itervalues())
361
just_parents.difference_update(parent_map)
362
# Ignore ghost parents
363
present_parents = self.repository.inventories.get_parent_map(
365
ghost_keys = just_parents.difference(present_parents)
366
needed_inventories = list(present_parents) + inventory_key_order
367
needed_inventories = [k[-1] for k in needed_inventories]
369
for inv in self.repository.iter_inventories(needed_inventories):
370
revision_id = inv.revision_id
372
as_bytes = inv_to_str(inv)
373
# The sha1 is validated as the xml/textual form, not as the
374
# form-in-the-repository
375
sha1 = osutils.sha_string(as_bytes)
376
as_lines = osutils.split_lines(as_bytes)
378
all_lines[key] = as_lines
379
if key in just_parents:
380
# We don't transmit those entries
382
# Create an mpdiff for this text, and add it to the output
383
parent_keys = parent_map[key]
384
# See the comment in VF.make_mpdiffs about how this effects
385
# ordering when there are ghosts present. I think we have a latent
387
parent_lines = [all_lines[p_key] for p_key in parent_keys
388
if p_key not in ghost_keys]
389
diff = multiparent.MultiParent.from_lines(
390
as_lines, parent_lines)
391
text = ''.join(diff.to_patch())
392
parent_ids = [k[-1] for k in parent_keys]
393
generator = _MPDiffInventoryGenerator(self.repository,
395
for revision_id, parent_ids, sha1, diff in generator.iter_diffs():
396
text = b''.join(diff.to_patch())
393
397
self.bundle.add_multiparent_record(text, sha1, parent_ids,
394
398
'inventory', revision_id, None)
434
438
for mpdiff, item_key, in zip(mpdiffs, ordered_keys):
435
439
sha1 = sha1s[item_key]
436
440
parents = [key[-1] for key in parent_map[item_key]]
437
text = ''.join(mpdiff.to_patch())
441
text = b''.join(mpdiff.to_patch())
438
442
# Infer file id records as appropriate.
439
443
if len(item_key) == 2:
440
444
file_id = item_key[0]
464
468
all into memory at once. Reading it into memory all at once is
465
469
(currently) faster.
467
repository.lock_write()
471
with repository.lock_write():
469
472
ri = RevisionInstaller(self.get_bundle_reader(stream_input),
470
473
self._serializer, repository)
471
474
return ri.install()
475
476
def get_merge_request(self, target_repo):
476
477
"""Provide data for performing a merge
615
parents = [prefix + (parent,) for parent in meta['parents']]
616
vf_records.append((key, parents, meta['sha1'], d_func(text)))
616
parents = [prefix + (parent,) for parent in meta[b'parents']]
617
vf_records.append((key, parents, meta[b'sha1'], d_func(text)))
617
618
versionedfile.add_mpdiffs(vf_records)
619
620
def _get_parent_inventory_texts(self, inventory_text_cache,
657
658
return parent_texts
659
660
def _install_inventory_records(self, records):
660
if (self._info['serializer'] == self._repository._serializer.format_num
661
if (self._info[b'serializer'] == self._repository._serializer.format_num
661
662
and self._repository._serializer.support_altered_by_hack):
662
663
return self._install_mp_records_keys(self._repository.inventories,
670
671
# inventory deltas to apply rather than calling add_inventory from
671
672
# scratch each time.
672
673
inventory_cache = lru_cache.LRUCache(10)
673
pb = ui.ui_factory.nested_progress_bar()
674
with ui.ui_factory.nested_progress_bar() as pb:
675
675
num_records = len(records)
676
676
for idx, (key, metadata, bytes) in enumerate(records):
677
677
pb.update('installing inventory', idx, num_records)
678
678
revision_id = key[-1]
679
parent_ids = metadata['parents']
679
parent_ids = metadata[b'parents']
680
680
# Note: This assumes the local ghosts are identical to the
681
681
# ghosts in the source, as the Bundle serialization
682
682
# format doesn't record ghosts.
688
688
# as lines and then cast back to a string.
689
689
target_lines = multiparent.MultiParent.from_patch(bytes
690
690
).to_lines(p_texts)
691
inv_text = ''.join(target_lines)
691
inv_text = b''.join(target_lines)
693
693
sha1 = osutils.sha_string(inv_text)
694
if sha1 != metadata['sha1']:
694
if sha1 != metadata[b'sha1']:
695
695
raise errors.BadBundle("Can't convert to target format")
696
696
# Add this to the cache so we don't have to extract it again.
697
697
inventory_text_cache[revision_id] = inv_text
712
712
except errors.UnsupportedInventoryKind:
713
713
raise errors.IncompatibleRevision(repr(self._repository))
714
714
inventory_cache[revision_id] = target_inv
718
716
def _handle_root(self, target_inv, parent_ids):
719
717
revision_id = target_inv.revision_id