To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to breezy/plugins/fastimport/revision_store.py
- browse files at revision 7029
- compare with another revision
- compare with revision 5200.3.2
- download diff
- download tarball
- view history from revision 7029
- Committer: Breezy landing bot
- Author(s): Jelmer Vernooij
- Date: 2018-07-04 12:50:55 UTC
- mfrom: (7027.2.8 git-fixes)
- Revision ID: breezy.the.bot@gmail.com-20180704125055-8nni25pn2439p48v
Fix eol handling in knits on Python 3, port fastimport plugin to Python 3.
Merged from https://code.launchpad.net/~jelmer/brz/fastimport-fixes/+merge/348924
Merged from https://code.launchpad.net/~jelmer/brz/fastimport-fixes/+merge/348924
- files modified:
- breezy/branch.py
added
removed
breezy/plugins/fastimport/revision_store.py
165
165
yield change
166
166
167
167
168
class AbstractRevisionStore(object):
168
class RevisionStore(object):
169
169
170
170
def __init__(self, repo):
171
171
"""An object responsible for loading revisions into a repository.
258
258
259
259
def get_parents_and_revision_for_entry(self, ie):
260
260
"""Get the parents and revision for an inventory entry.
261
261
262
262
:param ie: the inventory entry
263
263
:return parents, revision_id where
264
264
parents is the tuple of parent revision_ids for the per-file graph
299
299
if len(heads) > 1:
300
300
changed = True
301
301
elif (parent_entry.name != ie.name or parent_entry.kind != ie.kind or
302
parent_entry.parent_id != ie.parent_id):
302
parent_entry.parent_id != ie.parent_id):
303
303
changed = True
304
304
elif ie.kind == 'file':
305
305
if (parent_entry.text_sha1 != ie.text_sha1 or
314
314
rev_id = parent_entry.revision
315
315
return tuple(heads), rev_id
316
316
317
def load(self, rev, inv, signature, text_provider, parents_provider,
318
inventories_provider=None):
319
"""Load a revision.
320
321
:param rev: the Revision
322
:param inv: the inventory
323
:param signature: signing information
324
:param text_provider: a callable expecting a file_id parameter
325
that returns the text for that file-id
326
:param parents_provider: a callable expecting a file_id parameter
327
that return the list of parent-ids for that file-id
328
:param inventories_provider: a callable expecting a repository and
329
a list of revision-ids, that returns:
330
* the list of revision-ids present in the repository
331
* the list of inventories for the revision-id's,
332
including an empty inventory for the missing revisions
333
If None, a default implementation is provided.
334
"""
335
# NOTE: This is breezy.repository._install_revision refactored to
336
# to provide more flexibility in how previous revisions are cached,
337
# data is feed in, etc.
338
339
# Get the non-ghost parents and their inventories
340
if inventories_provider is None:
341
inventories_provider = self._default_inventories_provider
342
present_parents, parent_invs = inventories_provider(rev.parent_ids)
343
344
# Load the inventory
345
try:
346
rev.inventory_sha1 = self._add_inventory(rev.revision_id,
347
inv, present_parents, parent_invs)
348
except errors.RevisionAlreadyPresent:
349
pass
350
351
# Load the texts, signature and revision
352
entries = self._non_root_entries_iter(inv, rev.revision_id)
353
self._load_texts(rev.revision_id, entries, text_provider,
354
parents_provider)
355
if signature is not None:
356
self.repo.add_signature_text(rev.revision_id, signature)
357
self._add_revision(rev, inv)
358
359
317
def load_using_delta(self, rev, basis_inv, inv_delta, signature,
360
318
text_provider, parents_provider, inventories_provider=None):
361
319
"""Load a revision by applying a delta to a (CHK)Inventory.
434
392
if signature is not None:
435
393
raise AssertionError('signatures not guaranteed yet')
436
394
self.repo.add_signature_text(rev.revision_id, signature)
437
# self._add_revision(rev, inv)
438
395
return builder.revision_tree().root_inventory
439
396
440
def _non_root_entries_iter(self, inv, revision_id):
441
if hasattr(inv, 'iter_non_root_entries'):
442
entries = inv.iter_non_root_entries()
443
else:
444
path_entries = inv.iter_entries()
445
# Backwards compatibility hack: skip the root id.
446
if not self.repo.supports_rich_root():
447
path, root = next(path_entries)
448
if root.revision != revision_id:
449
raise errors.IncompatibleRevision(repr(self.repo))
450
entries = iter([ie for path, ie in path_entries])
451
return entries
452
453
def _load_texts(self, revision_id, entries, text_provider,
454
parents_provider):
455
"""Load texts to a repository for inventory entries.
456
457
This method is provided for subclasses to use or override.
458
459
:param revision_id: the revision identifier
460
:param entries: iterator over the inventory entries
461
:param text_provider: a callable expecting a file_id parameter
462
that returns the text for that file-id
463
:param parents_provider: a callable expecting a file_id parameter
464
that return the list of parent-ids for that file-id
465
"""
466
raise NotImplementedError(self._load_texts)
467
468
def _add_inventory(self, revision_id, inv, parents, parent_invs):
469
"""Add the inventory inv to the repository as revision_id.
470
471
:param parents: The revision ids of the parents that revision_id
472
is known to have and are in the repository already.
473
:param parent_invs: the parent inventories
474
475
:returns: The validator(which is a sha1 digest, though what is sha'd is
476
repository format specific) of the serialized inventory.
477
"""
478
return self.repo.add_inventory(revision_id, inv, parents)
479
480
def _add_inventory_by_delta(self, revision_id, basis_inv, inv_delta,
481
parents, parent_invs):
482
"""Add the inventory to the repository as revision_id.
483
484
:param basis_inv: the basis Inventory or CHKInventory
485
:param inv_delta: the inventory delta
486
:param parents: The revision ids of the parents that revision_id
487
is known to have and are in the repository already.
488
:param parent_invs: the parent inventories
489
490
:returns: (validator, inv) where validator is the validator
491
(which is a sha1 digest, though what is sha'd is repository format
492
specific) of the serialized inventory;
493
inv is the generated inventory
494
"""
495
if len(parents):
496
if self._supports_chks:
497
try:
498
validator, new_inv = self.repo.add_inventory_by_delta(parents[0],
499
inv_delta, revision_id, parents, basis_inv=basis_inv,
500
propagate_caches=False)
501
except errors.InconsistentDelta:
502
#print "BASIS INV IS\n%s\n" % "\n".join([str(i) for i in basis_inv.iter_entries_by_dir()])
503
trace.mutter("INCONSISTENT DELTA IS:\n%s\n" % "\n".join([str(i) for i in inv_delta]))
504
raise
505
else:
506
validator, new_inv = self.repo.add_inventory_by_delta(parents[0],
507
inv_delta, revision_id, parents)
508
else:
509
if isinstance(basis_inv, inventory.CHKInventory):
510
new_inv = basis_inv.create_by_apply_delta(inv_delta, revision_id)
511
else:
512
new_inv = inventory.Inventory(revision_id=revision_id)
513
# This is set in the delta so remove it to prevent a duplicate
514
new_inv.delete(inventory.ROOT_ID)
515
new_inv.apply_delta(inv_delta)
516
validator = self.repo.add_inventory(revision_id, new_inv, parents)
517
return validator, new_inv
518
519
def _add_revision(self, rev, inv):
520
"""Add a revision and its inventory to a repository.
521
522
:param rev: the Revision
523
:param inv: the inventory
524
"""
525
self.repo.add_revision(rev.revision_id, rev, inv)
526
527
def _default_inventories_provider(self, revision_ids):
528
"""An inventories provider that queries the repository."""
529
present = []
530
inventories = []
531
for revision_id in revision_ids:
532
if self.repo.has_revision(revision_id):
533
present.append(revision_id)
534
rev_tree = self.repo.revision_tree(revision_id)
535
else:
536
rev_tree = self.repo.revision_tree(None)
537
inventories.append(rev_tree.root_inventory)
538
return present, inventories
539
540
541
class RevisionStore1(AbstractRevisionStore):
542
"""A RevisionStore that uses the old breezy Repository API.
543
544
The old API was present until bzr.dev rev 3510.
545
"""
546
547
def _load_texts(self, revision_id, entries, text_provider, parents_provider):
548
"""See RevisionStore._load_texts()."""
549
# Add the texts that are not already present
550
tx = self.repo.get_transaction()
551
for ie in entries:
552
# This test is *really* slow: over 50% of import time
553
#w = self.repo.weave_store.get_weave_or_empty(ie.file_id, tx)
554
#if ie.revision in w:
555
# continue
556
# Try another way, realising that this assumes that the
557
# version is not already there. In the general case,
558
# a shared repository might already have the revision but
559
# we arguably don't need that check when importing from
560
# a foreign system.
561
if ie.revision != revision_id:
562
continue
563
file_id = ie.file_id
564
text_parents = [(file_id, p) for p in parents_provider(file_id)]
565
lines = text_provider(file_id)
566
vfile = self.repo.weave_store.get_weave_or_empty(file_id, tx)
567
vfile.add_lines(revision_id, text_parents, lines)
568
569
def get_file_lines(self, revision_id, file_id):
570
tx = self.repo.get_transaction()
571
w = self.repo.weave_store.get_weave(file_id, tx)
572
return w.get_lines(revision_id)
573
574
def _add_revision(self, rev, inv):
575
# There's no need to do everything repo.add_revision does and
576
# doing so (since bzr.dev 3392) can be pretty slow for long
577
# delta chains on inventories. Just do the essentials here ...
578
_mod_revision.check_not_reserved_id(rev.revision_id)
579
self.repo._revision_store.add_revision(rev, self.repo.get_transaction())
580
581
582
class RevisionStore2(AbstractRevisionStore):
583
"""A RevisionStore that uses the new breezy Repository API."""
584
585
def _load_texts(self, revision_id, entries, text_provider, parents_provider):
586
"""See RevisionStore._load_texts()."""
587
text_keys = {}
588
for ie in entries:
589
text_keys[(ie.file_id, ie.revision)] = ie
590
text_parent_map = self.repo.texts.get_parent_map(text_keys)
591
missing_texts = set(text_keys) - set(text_parent_map)
592
self._load_texts_for_file_rev_ids(missing_texts, text_provider,
593
parents_provider)
594
595
def _load_texts_for_file_rev_ids(self, file_rev_ids, text_provider,
596
parents_provider):
597
"""Load texts to a repository for file-ids, revision-id tuples.
598
599
:param file_rev_ids: iterator over the (file_id, revision_id) tuples
600
:param text_provider: a callable expecting a file_id parameter
601
that returns the text for that file-id
602
:param parents_provider: a callable expecting a file_id parameter
603
that return the list of parent-ids for that file-id
604
"""
605
for file_id, revision_id in file_rev_ids:
606
text_key = (file_id, revision_id)
607
text_parents = [(file_id, p) for p in parents_provider(file_id)]
608
lines = text_provider(file_id)
609
#print "adding text for %s\n\tparents:%s" % (text_key,text_parents)
610
self.repo.texts.add_lines(text_key, text_parents, lines)
611
612
397
def get_file_lines(self, revision_id, file_id):
613
398
record = next(self.repo.texts.get_record_stream([(file_id, revision_id)],
614
399
'unordered', True))
615
400
if record.storage_kind == 'absent':
616
401
raise errors.RevisionNotPresent(record.key, self.repo)
617
402
return osutils.split_lines(record.get_bytes_as('fulltext'))
618
619
# This is breaking imports into brisbane-core currently
620
#def _add_revision(self, rev, inv):
621
# # There's no need to do everything repo.add_revision does and
622
# # doing so (since bzr.dev 3392) can be pretty slow for long
623
# # delta chains on inventories. Just do the essentials here ...
624
# _mod_revision.check_not_reserved_id(rev.revision_id)
625
# self.repo._add_revision(rev)
626
627
628
class ImportRevisionStore1(RevisionStore1):
629
"""A RevisionStore (old Repository API) optimised for importing.
630
631
This implementation caches serialised inventory texts and provides
632
fine-grained control over when inventories are stored as fulltexts.
633
"""
634
635
def __init__(self, repo, parent_texts_to_cache=1, fulltext_when=None,
636
random_ids=True):
637
"""See AbstractRevisionStore.__init__.
638
639
:param repository: the target repository
640
:param parent_text_to_cache: the number of parent texts to cache
641
:para fulltext_when: if non None, a function to call to decide
642
whether to fulltext the inventory or not. The revision count
643
is passed as a parameter and the result is treated as a boolean.
644
"""
645
RevisionStore1.__init__(self, repo)
646
self.inv_parent_texts = lru_cache.LRUCache(parent_texts_to_cache)
647
self.fulltext_when = fulltext_when
648
self.random_ids = random_ids
649
self.revision_count = 0
650
651
def _add_inventory(self, revision_id, inv, parents, parent_invs):
652
"""See RevisionStore._add_inventory."""
653
# Code taken from breezy.repository.add_inventory
654
assert self.repo.is_in_write_group()
655
_mod_revision.check_not_reserved_id(revision_id)
656
assert inv.revision_id is None or inv.revision_id == revision_id, \
657
"Mismatch between inventory revision" \
658
" id and insertion revid (%r, %r)" % (inv.revision_id, revision_id)
659
assert inv.root is not None
660
inv_lines = self.repo._serialise_inventory_to_lines(inv)
661
inv_vf = self.repo.get_inventory_weave()
662
sha1, num_bytes, parent_text = self._inventory_add_lines(inv_vf,
663
revision_id, parents, inv_lines, self.inv_parent_texts)
664
self.inv_parent_texts[revision_id] = parent_text
665
return sha1
666
667
def _inventory_add_lines(self, inv_vf, version_id, parents, lines,
668
parent_texts):
669
"""See Repository._inventory_add_lines()."""
670
# setup parameters used in original code but not this API
671
self.revision_count += 1
672
if self.fulltext_when is not None:
673
delta = not self.fulltext_when(self.revision_count)
674
else:
675
delta = inv_vf.delta
676
left_matching_blocks = None
677
random_id = self.random_ids
678
check_content = False
679
680
# breezy.knit.add_lines() but error checking optimised
681
inv_vf._check_add(version_id, lines, random_id, check_content)
682
683
####################################################################
684
# breezy.knit._add() but skip checking if fulltext better than delta
685
####################################################################
686
687
line_bytes = ''.join(lines)
688
digest = osutils.sha_string(line_bytes)
689
present_parents = []
690
for parent in parents:
691
if inv_vf.has_version(parent):
692
present_parents.append(parent)
693
if parent_texts is None:
694
parent_texts = {}
695
696
# can only compress against the left most present parent.
697
if (delta and
698
(len(present_parents) == 0 or
699
present_parents[0] != parents[0])):
700
delta = False
701
702
text_length = len(line_bytes)
703
options = []
704
if lines:
705
if lines[-1][-1] != '\n':
706
# copy the contents of lines.
707
lines = lines[:]
708
options.append('no-eol')
709
lines[-1] = lines[-1] + '\n'
710
line_bytes += '\n'
711
712
#if delta:
713
# # To speed the extract of texts the delta chain is limited
714
# # to a fixed number of deltas. This should minimize both
715
# # I/O and the time spend applying deltas.
716
# delta = inv_vf._check_should_delta(present_parents)
717
718
assert isinstance(version_id, str)
719
content = inv_vf.factory.make(lines, version_id)
720
if delta or (inv_vf.factory.annotated and len(present_parents) > 0):
721
# Merge annotations from parent texts if needed.
722
delta_hunks = inv_vf._merge_annotations(content, present_parents,
723
parent_texts, delta, inv_vf.factory.annotated,
724
left_matching_blocks)
725
726
if delta:
727
options.append('line-delta')
728
store_lines = inv_vf.factory.lower_line_delta(delta_hunks)
729
size, bytes = inv_vf._data._record_to_data(version_id, digest,
730
store_lines)
731
else:
732
options.append('fulltext')
733
# isinstance is slower and we have no hierarchy.
734
if inv_vf.factory.__class__ == knit.KnitPlainFactory:
735
# Use the already joined bytes saving iteration time in
736
# _record_to_data.
737
size, bytes = inv_vf._data._record_to_data(version_id, digest,
738
lines, [line_bytes])
739
else:
740
# get mixed annotation + content and feed it into the
741
# serialiser.
742
store_lines = inv_vf.factory.lower_fulltext(content)
743
size, bytes = inv_vf._data._record_to_data(version_id, digest,
744
store_lines)
745
746
access_memo = inv_vf._data.add_raw_records([size], bytes)[0]
747
inv_vf._index.add_versions(
748
((version_id, options, access_memo, parents),),
749
random_id=random_id)
750
return digest, text_length, content
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0