1
# Copyright (C) 2009-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""InterRepository operations."""
19
from io import BytesIO
22
from dulwich.errors import (
25
from dulwich.object_store import (
26
ObjectStoreGraphWalker,
28
from dulwich.protocol import (
32
from dulwich.refs import (
36
from dulwich.walk import Walker
38
from ..errors import (
40
FetchLimitUnsupported,
43
NoRoundtrippingSupport,
46
from ..repository import (
50
from ..revision import (
64
DetermineWantsRecorder,
66
from .mapping import (
69
from .object_store import (
73
MissingObjectsIterator,
80
from .repository import (
89
from .unpeel_map import (
94
class InterToGitRepository(InterRepository):
95
"""InterRepository that copies into a Git repository."""
97
_matching_repo_format = GitRepositoryFormat()
99
def __init__(self, source, target):
100
super(InterToGitRepository, self).__init__(source, target)
101
self.mapping = self.target.get_mapping()
102
self.source_store = get_object_store(self.source, self.mapping)
105
def _get_repo_format_to_test():
108
def copy_content(self, revision_id=None, pb=None):
109
"""See InterRepository.copy_content."""
110
self.fetch(revision_id, pb, find_ghosts=False)
112
def fetch_refs(self, update_refs, lossy, overwrite=False):
113
"""Fetch possibly roundtripped revisions into the target repository
116
:param update_refs: Generate refs to fetch. Receives dictionary
117
with old refs (git shas), returns dictionary of new names to
119
:param lossy: Whether to roundtrip
120
:return: old refs, new refs
122
raise NotImplementedError(self.fetch_refs)
124
def search_missing_revision_ids(self,
125
find_ghosts=True, revision_ids=None,
126
if_present_ids=None, limit=None):
127
if limit is not None:
128
raise FetchLimitUnsupported(self)
132
todo.extend(revision_ids)
134
todo.extend(revision_ids)
135
with self.source_store.lock_read():
136
for revid in revision_ids:
137
if revid == NULL_REVISION:
140
git_sha = self.source_store._lookup_revision_sha1(revid)
142
raise NoSuchRevision(revid, self.source)
143
git_shas.append(git_sha)
148
sha for sha in self.target.controldir.get_refs_container().as_dict().values()
150
missing_revids = set()
152
for (kind, type_data) in self.source_store.lookup_git_sha(
155
missing_revids.add(type_data[0])
156
return self.source.revision_ids_to_search_result(missing_revids)
158
def _warn_slow(self):
159
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
161
'Pushing from a Bazaar to a Git repository. '
162
'For better performance, push into a Bazaar repository.')
165
class InterToLocalGitRepository(InterToGitRepository):
166
"""InterBranch implementation between a Bazaar and a Git repository."""
168
def __init__(self, source, target):
169
super(InterToLocalGitRepository, self).__init__(source, target)
170
self.target_store = self.target.controldir._git.object_store
171
self.target_refs = self.target.controldir._git.refs
173
def _commit_needs_fetching(self, sha_id):
175
return (sha_id not in self.target_store)
176
except NoSuchRevision:
180
def _revision_needs_fetching(self, sha_id, revid):
181
if revid == NULL_REVISION:
185
sha_id = self.source_store._lookup_revision_sha1(revid)
188
return self._commit_needs_fetching(sha_id)
190
def missing_revisions(self, stop_revisions):
191
"""Find the revisions that are missing from the target repository.
193
:param stop_revisions: Revisions to check for (tuples with
195
:return: sequence of missing revisions, in topological order
196
:raise: NoSuchRevision if the stop_revisions are not present in
201
for (sha1, revid) in stop_revisions:
202
if sha1 is not None and revid is not None:
203
revid_sha_map[revid] = sha1
204
stop_revids.append(revid)
205
elif sha1 is not None:
206
if self._commit_needs_fetching(sha1):
207
for (kind, (revid, tree_sha, verifiers)) in self.source_store.lookup_git_sha(sha1):
208
revid_sha_map[revid] = sha1
209
stop_revids.append(revid)
213
stop_revids.append(revid)
215
graph = self.source.get_graph()
216
with ui.ui_factory.nested_progress_bar() as pb:
219
for revid in stop_revids:
220
sha1 = revid_sha_map.get(revid)
221
if (revid not in missing and
222
self._revision_needs_fetching(sha1, revid)):
224
new_stop_revids.append(revid)
226
parent_map = graph.get_parent_map(new_stop_revids)
227
for parent_revids in parent_map.values():
228
stop_revids.update(parent_revids)
229
pb.update("determining revisions to fetch", len(missing))
230
return graph.iter_topo_order(missing)
232
def _get_target_bzr_refs(self):
233
"""Return a dictionary with references.
235
:return: Dictionary with reference names as keys and tuples
236
with Git SHA, Bazaar revid as values.
239
for k in self.target._git.refs.allkeys():
241
v = self.target._git.refs.read_ref(k)
246
if v and not v.startswith(SYMREF):
248
for (kind, type_data) in self.source_store.lookup_git_sha(
250
if kind == "commit" and self.source.has_revision(
256
bzr_refs[k] = (v, revid)
259
def fetch_refs(self, update_refs, lossy, overwrite=False):
262
with self.source_store.lock_read():
263
old_refs = self._get_target_bzr_refs()
264
new_refs = update_refs(old_refs)
265
revidmap = self.fetch_objects(
266
[(git_sha, bzr_revid)
267
for (git_sha, bzr_revid) in new_refs.values()
268
if git_sha is None or not git_sha.startswith(SYMREF)],
270
for name, (gitid, revid) in new_refs.items():
273
gitid = revidmap[revid][0]
275
gitid = self.source_store._lookup_revision_sha1(revid)
276
if gitid.startswith(SYMREF):
277
self.target_refs.set_symbolic_ref(
278
name, gitid[len(SYMREF):])
281
old_git_id = old_refs[name][0]
283
self.target_refs.add_if_new(name, gitid)
285
self.target_refs.set_if_equals(name, old_git_id, gitid)
286
result_refs[name] = (gitid, revid if not lossy else self.mapping.revision_id_foreign_to_bzr(gitid))
287
return revidmap, old_refs, result_refs
289
def fetch_objects(self, revs, lossy, limit=None):
290
if not lossy and not self.mapping.roundtripping:
291
for git_sha, bzr_revid in revs:
292
if (bzr_revid is not None and
293
needs_roundtripping(self.source, bzr_revid)):
294
raise NoPushSupport(self.source, self.target, self.mapping,
296
with self.source_store.lock_read():
297
todo = list(self.missing_revisions(revs))[:limit]
299
with ui.ui_factory.nested_progress_bar() as pb:
300
object_generator = MissingObjectsIterator(
301
self.source_store, self.source, pb)
302
for (old_revid, git_sha) in object_generator.import_revisions(
305
new_revid = self.mapping.revision_id_foreign_to_bzr(
308
new_revid = old_revid
310
self.mapping.revision_id_bzr_to_foreign(old_revid)
311
except InvalidRevisionId:
313
revidmap[old_revid] = (git_sha, new_revid)
314
self.target_store.add_objects(object_generator)
317
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
318
fetch_spec=None, mapped_refs=None, lossy=False):
319
if mapped_refs is not None:
320
stop_revisions = mapped_refs
321
elif revision_id is not None:
322
stop_revisions = [(None, revision_id)]
323
elif fetch_spec is not None:
324
recipe = fetch_spec.get_recipe()
325
if recipe[0] in ("search", "proxy-search"):
326
stop_revisions = [(None, revid) for revid in recipe[1]]
328
raise AssertionError(
329
"Unsupported search result type %s" % recipe[0])
331
stop_revisions = [(None, revid)
332
for revid in self.source.all_revision_ids()]
335
revidmap = self.fetch_objects(stop_revisions, lossy=lossy)
336
except NoPushSupport:
337
raise NoRoundtrippingSupport(self.source, self.target)
338
return FetchResult(revidmap)
341
def is_compatible(source, target):
342
"""Be compatible with GitRepository."""
343
return (not isinstance(source, GitRepository) and
344
isinstance(target, LocalGitRepository))
347
class InterToRemoteGitRepository(InterToGitRepository):
349
def fetch_refs(self, update_refs, lossy, overwrite=False):
350
"""Import the gist of the ancestry of a particular revision."""
351
if not lossy and not self.mapping.roundtripping:
352
raise NoPushSupport(self.source, self.target, self.mapping)
353
unpeel_map = UnpeelMap.from_repository(self.source)
356
def git_update_refs(old_refs):
359
k: (v, None) for (k, v) in old_refs.items()}
360
new_refs = update_refs(self.old_refs)
361
for name, (gitid, revid) in new_refs.items():
363
git_sha = self.source_store._lookup_revision_sha1(revid)
364
gitid = unpeel_map.re_unpeel_tag(
365
git_sha, old_refs.get(name))
367
if remote_divergence(
368
old_refs.get(name), gitid, self.source_store):
369
raise DivergedBranches(self.source, self.target)
373
with self.source_store.lock_read():
374
result = self.target.send_pack(
375
git_update_refs, self.source_store.generate_lossy_pack_data)
376
if result is not None and not isinstance(result, dict):
377
for ref, error in result.ref_status.items():
379
raise RemoteGitError(
380
'unable to update ref %r: %s' % (ref, error))
381
new_refs = result.refs
382
else: # dulwich < 0.20.3
385
return revidmap, self.old_refs, new_refs
388
def is_compatible(source, target):
389
"""Be compatible with GitRepository."""
390
return (not isinstance(source, GitRepository) and
391
isinstance(target, RemoteGitRepository))
394
class GitSearchResult(object):
396
def __init__(self, start, exclude, keys):
398
self._exclude = exclude
404
def get_recipe(self):
405
return ('search', self._start, self._exclude, len(self._keys))
408
class InterFromGitRepository(InterRepository):
410
_matching_repo_format = GitRepositoryFormat()
412
def _target_has_shas(self, shas):
413
raise NotImplementedError(self._target_has_shas)
415
def get_determine_wants_heads(self, wants, include_tags=False, tag_selector=None):
418
def determine_wants(refs):
420
for k, v in refs.items():
421
if k.endswith(ANNOTATED_TAG_SUFFIX):
422
unpeel_lookup[v] = refs[k[:-len(ANNOTATED_TAG_SUFFIX)]]
423
potential = set([unpeel_lookup.get(w, w) for w in wants])
425
for k, sha in refs.items():
426
if k.endswith(ANNOTATED_TAG_SUFFIX):
429
tag_name = ref_to_tag_name(k)
432
if tag_selector and not tag_selector(tag_name):
437
return list(potential - self._target_has_shas(potential))
438
return determine_wants
440
def determine_wants_all(self, refs):
441
raise NotImplementedError(self.determine_wants_all)
444
def _get_repo_format_to_test():
447
def copy_content(self, revision_id=None):
448
"""See InterRepository.copy_content."""
449
self.fetch(revision_id, find_ghosts=False)
451
def search_missing_revision_ids(self,
452
find_ghosts=True, revision_ids=None,
453
if_present_ids=None, limit=None):
454
if limit is not None:
455
raise FetchLimitUnsupported(self)
456
if revision_ids is None and if_present_ids is None:
457
todo = set(self.source.all_revision_ids())
460
if revision_ids is not None:
461
for revid in revision_ids:
462
if not self.source.has_revision(revid):
463
raise NoSuchRevision(revid, self.source)
464
todo.update(revision_ids)
465
if if_present_ids is not None:
466
todo.update(if_present_ids)
467
result_set = todo.difference(self.target.all_revision_ids())
468
result_parents = set(itertools.chain.from_iterable(
469
self.source.get_graph().get_parent_map(result_set).values()))
470
included_keys = result_set.intersection(result_parents)
471
start_keys = result_set.difference(included_keys)
472
exclude_keys = result_parents.difference(result_set)
473
return GitSearchResult(start_keys, exclude_keys, result_set)
476
class InterGitNonGitRepository(InterFromGitRepository):
477
"""Base InterRepository that copies revisions from a Git into a non-Git
480
def _target_has_shas(self, shas):
484
revid = self.source.lookup_foreign_revision_id(sha)
485
except NotCommitError:
486
# Commit is definitely not present
490
return set([revids[r] for r in self.target.has_revisions(revids)])
492
def determine_wants_all(self, refs):
494
for k, v in refs.items():
495
# For non-git target repositories, only worry about peeled
498
potential.add(self.source.controldir.get_peeled(k) or v)
499
return list(potential - self._target_has_shas(potential))
501
def _warn_slow(self):
502
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
504
'Fetching from Git to Bazaar repository. '
505
'For better performance, fetch into a Git repository.')
507
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
508
"""Fetch objects from a remote server.
510
:param determine_wants: determine_wants callback
511
:param mapping: BzrGitMapping to use
512
:param limit: Maximum number of commits to import.
513
:return: Tuple with pack hint, last imported revision id and remote
516
raise NotImplementedError(self.fetch_objects)
518
def get_determine_wants_revids(self, revids, include_tags=False, tag_selector=None):
520
for revid in set(revids):
521
if self.target.has_revision(revid):
523
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
525
return self.get_determine_wants_heads(
526
wants, include_tags=include_tags, tag_selector=tag_selector)
528
def fetch(self, revision_id=None, find_ghosts=False,
529
mapping=None, fetch_spec=None, include_tags=False, lossy=False):
531
mapping = self.source.get_mapping()
532
if revision_id is not None:
533
interesting_heads = [revision_id]
534
elif fetch_spec is not None:
535
recipe = fetch_spec.get_recipe()
536
if recipe[0] in ("search", "proxy-search"):
537
interesting_heads = recipe[1]
539
raise AssertionError("Unsupported search result type %s" %
542
interesting_heads = None
544
if interesting_heads is not None:
545
determine_wants = self.get_determine_wants_revids(
546
interesting_heads, include_tags=include_tags)
548
determine_wants = self.determine_wants_all
550
(pack_hint, _, remote_refs) = self.fetch_objects(
551
determine_wants, mapping, lossy=lossy)
552
if pack_hint is not None and self.target._format.pack_compresses:
553
self.target.pack(hint=pack_hint)
554
result = FetchResult()
555
result.refs = remote_refs
559
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
560
"""InterRepository that copies revisions from a remote Git into a non-Git
563
def get_target_heads(self):
564
# FIXME: This should be more efficient
565
all_revs = self.target.all_revision_ids()
566
parent_map = self.target.get_parent_map(all_revs)
568
for values in parent_map.values():
569
all_parents.update(values)
570
return set(all_revs) - all_parents
572
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
573
"""See `InterGitNonGitRepository`."""
575
store = get_object_store(self.target, mapping)
576
with store.lock_write():
577
heads = self.get_target_heads()
578
graph_walker = ObjectStoreGraphWalker(
579
[store._lookup_revision_sha1(head) for head in heads],
580
lambda sha: store[sha].parents)
581
wants_recorder = DetermineWantsRecorder(determine_wants)
583
with ui.ui_factory.nested_progress_bar() as pb:
584
objects_iter = self.source.fetch_objects(
585
wants_recorder, graph_walker, store.get_raw)
586
trace.mutter("Importing %d new revisions",
587
len(wants_recorder.wants))
588
(pack_hint, last_rev) = import_git_objects(
589
self.target, mapping, objects_iter, store,
590
wants_recorder.wants, pb, limit)
591
return (pack_hint, last_rev, wants_recorder.remote_refs)
594
def is_compatible(source, target):
595
"""Be compatible with GitRepository."""
596
if not isinstance(source, RemoteGitRepository):
598
if not target.supports_rich_root():
600
if isinstance(target, GitRepository):
602
if not getattr(target._format, "supports_full_versioned_files", True):
607
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
608
"""InterRepository that copies revisions from a local Git into a non-Git
611
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
612
"""See `InterGitNonGitRepository`."""
614
remote_refs = self.source.controldir.get_refs_container().as_dict()
615
wants = determine_wants(remote_refs)
616
target_git_object_retriever = get_object_store(self.target, mapping)
617
with ui.ui_factory.nested_progress_bar() as pb:
618
target_git_object_retriever.lock_write()
620
(pack_hint, last_rev) = import_git_objects(
621
self.target, mapping, self.source._git.object_store,
622
target_git_object_retriever, wants, pb, limit)
623
return (pack_hint, last_rev, remote_refs)
625
target_git_object_retriever.unlock()
628
def is_compatible(source, target):
629
"""Be compatible with GitRepository."""
630
if not isinstance(source, LocalGitRepository):
632
if not target.supports_rich_root():
634
if isinstance(target, GitRepository):
636
if not getattr(target._format, "supports_full_versioned_files", True):
641
class InterGitGitRepository(InterFromGitRepository):
642
"""InterRepository that copies between Git repositories."""
644
def fetch_refs(self, update_refs, lossy, overwrite=False):
646
raise LossyPushToSameVCS(self.source, self.target)
647
old_refs = self.target.controldir.get_refs_container()
650
def determine_wants(heads):
651
old_refs = dict([(k, (v, None))
652
for (k, v) in heads.as_dict().items()])
653
new_refs = update_refs(old_refs)
654
ref_changes.update(new_refs)
655
return [sha1 for (sha1, bzr_revid) in new_refs.values()]
656
self.fetch_objects(determine_wants, lossy=lossy)
657
for k, (git_sha, bzr_revid) in ref_changes.items():
658
self.target._git.refs[k] = git_sha
659
new_refs = self.target.controldir.get_refs_container()
660
return None, old_refs, new_refs
662
def fetch_objects(self, determine_wants, mapping=None, limit=None,
664
raise NotImplementedError(self.fetch_objects)
666
def _target_has_shas(self, shas):
668
[sha for sha in shas if sha in self.target._git.object_store])
670
def fetch(self, revision_id=None, find_ghosts=False,
671
mapping=None, fetch_spec=None, branches=None, limit=None,
672
include_tags=False, lossy=False):
674
mapping = self.source.get_mapping()
675
if revision_id is not None:
677
elif fetch_spec is not None:
678
recipe = fetch_spec.get_recipe()
679
if recipe[0] in ("search", "proxy-search"):
682
raise AssertionError(
683
"Unsupported search result type %s" % recipe[0])
685
if branches is not None:
686
determine_wants = self.get_determine_wants_branches(
687
branches, include_tags=include_tags)
688
elif fetch_spec is None and revision_id is None:
689
determine_wants = self.determine_wants_all
691
determine_wants = self.get_determine_wants_revids(
692
args, include_tags=include_tags)
693
wants_recorder = DetermineWantsRecorder(determine_wants)
694
self.fetch_objects(wants_recorder, mapping, limit=limit, lossy=lossy)
695
result = FetchResult()
696
result.refs = wants_recorder.remote_refs
699
def get_determine_wants_revids(self, revids, include_tags=False, tag_selector=None):
701
for revid in set(revids):
702
if revid == NULL_REVISION:
704
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
706
return self.get_determine_wants_heads(wants, include_tags=include_tags, tag_selector=tag_selector)
708
def get_determine_wants_branches(self, branches, include_tags=False):
709
def determine_wants(refs):
711
for name, value in refs.items():
712
if value == ZERO_SHA:
715
if name.endswith(ANNOTATED_TAG_SUFFIX):
718
if name in branches or (include_tags and is_tag(name)):
721
return determine_wants
723
def determine_wants_all(self, refs):
725
v for k, v in refs.items()
726
if not v == ZERO_SHA and not k.endswith(ANNOTATED_TAG_SUFFIX)])
727
return list(potential - self._target_has_shas(potential))
730
class InterLocalGitLocalGitRepository(InterGitGitRepository):
732
def fetch_objects(self, determine_wants, mapping=None, limit=None,
735
raise LossyPushToSameVCS(self.source, self.target)
736
if limit is not None:
737
raise FetchLimitUnsupported(self)
738
from .remote import DefaultProgressReporter
739
with ui.ui_factory.nested_progress_bar() as pb:
740
progress = DefaultProgressReporter(pb).progress
741
refs = self.source._git.fetch(
742
self.target._git, determine_wants,
744
return (None, None, refs)
747
def is_compatible(source, target):
748
"""Be compatible with GitRepository."""
749
return (isinstance(source, LocalGitRepository) and
750
isinstance(target, LocalGitRepository))
753
class InterRemoteGitLocalGitRepository(InterGitGitRepository):
755
def fetch_objects(self, determine_wants, mapping=None, limit=None,
758
raise LossyPushToSameVCS(self.source, self.target)
759
if limit is not None:
760
raise FetchLimitUnsupported(self)
761
graphwalker = self.target._git.get_graph_walker()
762
if (CAPABILITY_THIN_PACK in
763
self.source.controldir._client._fetch_capabilities):
764
# TODO(jelmer): Avoid reading entire file into memory and
765
# only processing it after the whole file has been fetched.
771
self.target._git.object_store.move_in_thin_pack(f)
776
f, commit, abort = self.target._git.object_store.add_pack()
778
refs = self.source.controldir.fetch_pack(
779
determine_wants, graphwalker, f.write)
781
return (None, None, refs)
782
except BaseException:
787
def is_compatible(source, target):
788
"""Be compatible with GitRepository."""
789
return (isinstance(source, RemoteGitRepository) and
790
isinstance(target, LocalGitRepository))
added
removed
breezy/git/interrepo.py
