To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to fetch.py
- browse files at revision 0.429.25
- compare with another revision
- compare with revision 5200.3.2
- download diff
- download tarball
- view history from revision 0.429.25
- Committer: Jelmer Vernooij
- Date: 2018-05-05 18:26:04 UTC
- mfrom: (0.200.1933 work)
- mto: (0.200.1934 work)
- mto: This revision was merged to the branch mainline in revision 6960.
- Revision ID: jelmer@jelmer.uk-20180505182604-rf3zyekbhwhlwddi
Merge trunk.
- files added:
- interrepo.py
- files modified:
- __init__.py
added
removed
fetch.py
18
18
19
19
from __future__ import absolute_import
20
20
21
from dulwich.errors import (
22
NotCommitError,
23
)
24
21
from dulwich.objects import (
25
22
Commit,
26
23
Tag,
30
27
ZERO_SHA,
31
28
)
32
29
from dulwich.object_store import (
33
ObjectStoreGraphWalker,
34
30
tree_lookup_path,
35
31
)
36
from dulwich.protocol import CAPABILITY_THIN_PACK
37
32
from dulwich.walk import Walker
38
33
from itertools import (
39
34
imap,
40
35
)
41
from io import BytesIO
42
36
import posixpath
43
import re
44
37
import stat
45
38
46
39
from ... import (
83
76
warn_unusual_mode,
84
77
)
85
78
from .object_store import (
86
BazaarObjectStore,
87
79
LRUTreeCache,
88
80
_tree_to_objects,
89
81
)
190
182
191
183
class SubmodulesRequireSubtrees(BzrError):
192
184
_fmt = ("The repository you are fetching from contains submodules, "
193
"which are not yet supported.")
185
"which require a Bazaar format that supports tree references.")
194
186
internal = False
195
187
196
188
332
324
if base_tree is not None and type(base_tree) is Tree:
333
325
invdelta.extend(remove_disappeared_children(base_bzr_tree, old_path,
334
326
base_tree, existing_children, lookup_object))
335
store_updater.add_object(tree, (file_id, ), path)
327
store_updater.add_object(tree, (file_id, revision_id), path)
336
328
return invdelta, child_modes
337
329
338
330
540
532
return pack_hints, last_imported
541
533
542
534
543
class InterFromGitRepository(InterRepository):
544
545
_matching_repo_format = GitRepositoryFormat()
546
547
def _target_has_shas(self, shas):
548
raise NotImplementedError(self._target_has_shas)
549
550
def get_determine_wants_heads(self, wants, include_tags=False):
551
wants = set(wants)
552
def determine_wants(refs):
553
potential = set(wants)
554
if include_tags:
555
for k, unpeeled in refs.iteritems():
556
if k.endswith("^{}"):
557
continue
558
if not is_tag(k):
559
continue
560
if unpeeled == ZERO_SHA:
561
continue
562
potential.add(unpeeled)
563
return list(potential - self._target_has_shas(potential))
564
return determine_wants
565
566
def determine_wants_all(self, refs):
567
raise NotImplementedError(self.determine_wants_all)
568
569
@staticmethod
570
def _get_repo_format_to_test():
571
return None
572
573
def copy_content(self, revision_id=None):
574
"""See InterRepository.copy_content."""
575
self.fetch(revision_id, find_ghosts=False)
576
577
def search_missing_revision_ids(self,
578
find_ghosts=True, revision_ids=None, if_present_ids=None,
579
limit=None):
580
if limit is not None:
581
raise errors.FetchLimitUnsupported(self)
582
git_shas = []
583
todo = []
584
if revision_ids:
585
todo.extend(revision_ids)
586
if if_present_ids:
587
todo.extend(revision_ids)
588
for revid in revision_ids:
589
if revid == NULL_REVISION:
590
continue
591
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
592
git_shas.append(git_sha)
593
walker = Walker(self.source._git.object_store,
594
include=git_shas, exclude=[
595
sha for sha in self.target.controldir.get_refs_container().as_dict().values()
596
if sha != ZERO_SHA])
597
missing_revids = set()
598
for entry in walker:
599
missing_revids.add(self.source.lookup_foreign_revision_id(entry.commit.id))
600
return self.source.revision_ids_to_search_result(missing_revids)
601
602
603
class InterGitNonGitRepository(InterFromGitRepository):
604
"""Base InterRepository that copies revisions from a Git into a non-Git
605
repository."""
606
607
def _target_has_shas(self, shas):
608
revids = {}
609
for sha in shas:
610
try:
611
revid = self.source.lookup_foreign_revision_id(sha)
612
except NotCommitError:
613
# Commit is definitely not present
614
continue
615
else:
616
revids[revid] = sha
617
return set([revids[r] for r in self.target.has_revisions(revids)])
618
619
def determine_wants_all(self, refs):
620
potential = set()
621
for k, v in refs.iteritems():
622
# For non-git target repositories, only worry about peeled
623
if v == ZERO_SHA:
624
continue
625
potential.add(self.source.controldir.get_peeled(k) or v)
626
return list(potential - self._target_has_shas(potential))
627
628
def get_determine_wants_heads(self, wants, include_tags=False):
629
wants = set(wants)
630
def determine_wants(refs):
631
potential = set(wants)
632
if include_tags:
633
for k, unpeeled in refs.iteritems():
634
if not is_tag(k):
635
continue
636
if unpeeled == ZERO_SHA:
637
continue
638
potential.add(self.source.controldir.get_peeled(k) or unpeeled)
639
return list(potential - self._target_has_shas(potential))
640
return determine_wants
641
642
def _warn_slow(self):
643
trace.warning(
644
'Fetching from Git to Bazaar repository. '
645
'For better performance, fetch into a Git repository.')
646
647
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
648
"""Fetch objects from a remote server.
649
650
:param determine_wants: determine_wants callback
651
:param mapping: BzrGitMapping to use
652
:param limit: Maximum number of commits to import.
653
:return: Tuple with pack hint, last imported revision id and remote refs
654
"""
655
raise NotImplementedError(self.fetch_objects)
656
657
def get_determine_wants_revids(self, revids, include_tags=False):
658
wants = set()
659
for revid in set(revids):
660
if self.target.has_revision(revid):
661
continue
662
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
663
wants.add(git_sha)
664
return self.get_determine_wants_heads(wants, include_tags=include_tags)
665
666
def fetch(self, revision_id=None, find_ghosts=False,
667
mapping=None, fetch_spec=None, include_tags=False):
668
if mapping is None:
669
mapping = self.source.get_mapping()
670
if revision_id is not None:
671
interesting_heads = [revision_id]
672
elif fetch_spec is not None:
673
recipe = fetch_spec.get_recipe()
674
if recipe[0] in ("search", "proxy-search"):
675
interesting_heads = recipe[1]
676
else:
677
raise AssertionError("Unsupported search result type %s" %
678
recipe[0])
679
else:
680
interesting_heads = None
681
682
if interesting_heads is not None:
683
determine_wants = self.get_determine_wants_revids(
684
interesting_heads, include_tags=include_tags)
685
else:
686
determine_wants = self.determine_wants_all
687
688
(pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
689
mapping)
690
if pack_hint is not None and self.target._format.pack_compresses:
691
self.target.pack(hint=pack_hint)
692
return remote_refs
693
694
695
_GIT_PROGRESS_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
696
def report_git_progress(pb, text):
697
text = text.rstrip("\r\n")
698
trace.mutter('git: %s', text)
699
g = _GIT_PROGRESS_RE.match(text)
700
if g is not None:
701
(text, pct, current, total) = g.groups()
702
pb.update(text, int(current), int(total))
703
else:
704
pb.update(text, 0, 0)
705
706
707
535
class DetermineWantsRecorder(object):
708
536
709
537
def __init__(self, actual):
719
547
return self.wants
720
548
721
549
722
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
723
"""InterRepository that copies revisions from a remote Git into a non-Git
724
repository."""
725
726
def get_target_heads(self):
727
# FIXME: This should be more efficient
728
all_revs = self.target.all_revision_ids()
729
parent_map = self.target.get_parent_map(all_revs)
730
all_parents = set()
731
map(all_parents.update, parent_map.itervalues())
732
return set(all_revs) - all_parents
733
734
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
735
"""See `InterGitNonGitRepository`."""
736
self._warn_slow()
737
store = BazaarObjectStore(self.target, mapping)
738
with store.lock_write():
739
heads = self.get_target_heads()
740
graph_walker = ObjectStoreGraphWalker(
741
[store._lookup_revision_sha1(head) for head in heads],
742
lambda sha: store[sha].parents)
743
wants_recorder = DetermineWantsRecorder(determine_wants)
744
745
pb = ui.ui_factory.nested_progress_bar()
746
try:
747
objects_iter = self.source.fetch_objects(
748
wants_recorder, graph_walker, store.get_raw,
749
progress=lambda text: report_git_progress(pb, text),)
750
trace.mutter("Importing %d new revisions",
751
len(wants_recorder.wants))
752
(pack_hint, last_rev) = import_git_objects(self.target,
753
mapping, objects_iter, store, wants_recorder.wants, pb,
754
limit)
755
return (pack_hint, last_rev, wants_recorder.remote_refs)
756
finally:
757
pb.finished()
758
759
@staticmethod
760
def is_compatible(source, target):
761
"""Be compatible with GitRepository."""
762
if not isinstance(source, RemoteGitRepository):
763
return False
764
if not target.supports_rich_root():
765
return False
766
if isinstance(target, GitRepository):
767
return False
768
if not getattr(target._format, "supports_full_versioned_files", True):
769
return False
770
return True
771
772
773
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
774
"""InterRepository that copies revisions from a local Git into a non-Git
775
repository."""
776
777
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
778
"""See `InterGitNonGitRepository`."""
779
self._warn_slow()
780
remote_refs = self.source.controldir.get_refs_container().as_dict()
781
wants = determine_wants(remote_refs)
782
create_pb = None
783
pb = ui.ui_factory.nested_progress_bar()
784
target_git_object_retriever = BazaarObjectStore(self.target, mapping)
785
try:
786
target_git_object_retriever.lock_write()
787
try:
788
(pack_hint, last_rev) = import_git_objects(self.target,
789
mapping, self.source._git.object_store,
790
target_git_object_retriever, wants, pb, limit)
791
return (pack_hint, last_rev, remote_refs)
792
finally:
793
target_git_object_retriever.unlock()
794
finally:
795
pb.finished()
796
797
@staticmethod
798
def is_compatible(source, target):
799
"""Be compatible with GitRepository."""
800
if not isinstance(source, LocalGitRepository):
801
return False
802
if not target.supports_rich_root():
803
return False
804
if isinstance(target, GitRepository):
805
return False
806
if not getattr(target._format, "supports_full_versioned_files", True):
807
return False
808
return True
809
810
811
class InterGitGitRepository(InterFromGitRepository):
812
"""InterRepository that copies between Git repositories."""
813
814
def fetch_refs(self, update_refs, lossy=False):
815
if lossy:
816
raise errors.LossyPushToSameVCS(self.source, self.target)
817
old_refs = self.target.controldir.get_refs_container()
818
ref_changes = {}
819
def determine_wants(heads):
820
old_refs = dict([(k, (v, None)) for (k, v) in heads.as_dict().iteritems()])
821
new_refs = update_refs(old_refs)
822
ref_changes.update(new_refs)
823
return [sha1 for (sha1, bzr_revid) in new_refs.itervalues()]
824
self.fetch_objects(determine_wants, lossy=lossy)
825
for k, (git_sha, bzr_revid) in ref_changes.iteritems():
826
self.target._git.refs[k] = git_sha
827
new_refs = self.target.controldir.get_refs_container()
828
return None, old_refs, new_refs
829
830
def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
831
if lossy:
832
raise errors.LossyPushToSameVCS(self.source, self.target)
833
if limit is not None:
834
raise errors.FetchLimitUnsupported(self)
835
graphwalker = self.target._git.get_graph_walker()
836
if (isinstance(self.source, LocalGitRepository) and
837
isinstance(self.target, LocalGitRepository)):
838
pb = ui.ui_factory.nested_progress_bar()
839
try:
840
refs = self.source._git.fetch(self.target._git, determine_wants,
841
lambda text: report_git_progress(pb, text))
842
finally:
843
pb.finished()
844
return (None, None, refs)
845
elif (isinstance(self.source, LocalGitRepository) and
846
isinstance(self.target, RemoteGitRepository)):
847
raise NotImplementedError
848
elif (isinstance(self.source, RemoteGitRepository) and
849
isinstance(self.target, LocalGitRepository)):
850
pb = ui.ui_factory.nested_progress_bar()
851
try:
852
if CAPABILITY_THIN_PACK in self.source.controldir._client._fetch_capabilities:
853
# TODO(jelmer): Avoid reading entire file into memory and
854
# only processing it after the whole file has been fetched.
855
f = BytesIO()
856
857
def commit():
858
if f.tell():
859
f.seek(0)
860
self.target._git.object_store.move_in_thin_pack(f)
861
862
def abort():
863
pass
864
else:
865
f, commit, abort = self.target._git.object_store.add_pack()
866
try:
867
refs = self.source.controldir.fetch_pack(
868
determine_wants, graphwalker, f.write,
869
lambda text: report_git_progress(pb, text))
870
commit()
871
return (None, None, refs)
872
except BaseException:
873
abort()
874
raise
875
finally:
876
pb.finished()
877
else:
878
raise AssertionError("fetching between %r and %r not supported" %
879
(self.source, self.target))
880
881
def _target_has_shas(self, shas):
882
return set([sha for sha in shas if sha in self.target._git.object_store])
883
884
def fetch(self, revision_id=None, find_ghosts=False,
885
mapping=None, fetch_spec=None, branches=None, limit=None, include_tags=False):
886
if mapping is None:
887
mapping = self.source.get_mapping()
888
r = self.target._git
889
if revision_id is not None:
890
args = [revision_id]
891
elif fetch_spec is not None:
892
recipe = fetch_spec.get_recipe()
893
if recipe[0] in ("search", "proxy-search"):
894
heads = recipe[1]
895
else:
896
raise AssertionError(
897
"Unsupported search result type %s" % recipe[0])
898
args = heads
899
if branches is not None:
900
def determine_wants(refs):
901
ret = []
902
for name, value in refs.iteritems():
903
if value == ZERO_SHA:
904
continue
905
906
if name in branches or (include_tags and is_tag(name)):
907
ret.append(value)
908
return ret
909
elif fetch_spec is None and revision_id is None:
910
determine_wants = self.determine_wants_all
911
else:
912
determine_wants = self.get_determine_wants_revids(args, include_tags=include_tags)
913
wants_recorder = DetermineWantsRecorder(determine_wants)
914
self.fetch_objects(wants_recorder, mapping, limit=limit)
915
return wants_recorder.remote_refs
916
917
@staticmethod
918
def is_compatible(source, target):
919
"""Be compatible with GitRepository."""
920
return (isinstance(source, GitRepository) and
921
isinstance(target, GitRepository))
922
923
def get_determine_wants_revids(self, revids, include_tags=False):
924
wants = set()
925
for revid in set(revids):
926
if self.target.has_revision(revid):
927
continue
928
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
929
wants.add(git_sha)
930
return self.get_determine_wants_heads(wants, include_tags=include_tags)
931
932
def determine_wants_all(self, refs):
933
potential = set([v for v in refs.values() if not v == ZERO_SHA])
934
return list(potential - self._target_has_shas(potential))
550
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0