1
# Copyright (C) 2009-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""InterRepository operations."""
19
from __future__ import absolute_import
21
from io import BytesIO
24
from dulwich.errors import (
27
from dulwich.object_store import (
28
ObjectStoreGraphWalker,
30
from dulwich.protocol import (
34
from dulwich.refs import (
38
from dulwich.walk import Walker
40
from ..errors import (
42
FetchLimitUnsupported,
45
NoRoundtrippingSupport,
48
from ..repository import (
52
from ..revision import (
55
from ..sixish import (
70
DetermineWantsRecorder,
72
from .mapping import (
75
from .object_store import (
79
MissingObjectsIterator,
86
from .repository import (
94
from .unpeel_map import (
99
class InterToGitRepository(InterRepository):
100
"""InterRepository that copies into a Git repository."""
102
_matching_repo_format = GitRepositoryFormat()
104
def __init__(self, source, target):
105
super(InterToGitRepository, self).__init__(source, target)
106
self.mapping = self.target.get_mapping()
107
self.source_store = get_object_store(self.source, self.mapping)
110
def _get_repo_format_to_test():
113
def copy_content(self, revision_id=None, pb=None):
114
"""See InterRepository.copy_content."""
115
self.fetch(revision_id, pb, find_ghosts=False)
117
def fetch_refs(self, update_refs, lossy, overwrite=False):
118
"""Fetch possibly roundtripped revisions into the target repository
121
:param update_refs: Generate refs to fetch. Receives dictionary
122
with old refs (git shas), returns dictionary of new names to
124
:param lossy: Whether to roundtrip
125
:return: old refs, new refs
127
raise NotImplementedError(self.fetch_refs)
129
def search_missing_revision_ids(self,
130
find_ghosts=True, revision_ids=None,
131
if_present_ids=None, limit=None):
132
if limit is not None:
133
raise FetchLimitUnsupported(self)
137
todo.extend(revision_ids)
139
todo.extend(revision_ids)
140
with self.source_store.lock_read():
141
for revid in revision_ids:
142
if revid == NULL_REVISION:
145
git_sha = self.source_store._lookup_revision_sha1(revid)
147
raise NoSuchRevision(revid, self.source)
148
git_shas.append(git_sha)
153
sha for sha in self.target.controldir.get_refs_container().as_dict().values()
155
missing_revids = set()
157
for (kind, type_data) in self.source_store.lookup_git_sha(
160
missing_revids.add(type_data[0])
161
return self.source.revision_ids_to_search_result(missing_revids)
163
def _warn_slow(self):
164
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
166
'Pushing from a Bazaar to a Git repository. '
167
'For better performance, push into a Bazaar repository.')
170
class InterToLocalGitRepository(InterToGitRepository):
171
"""InterBranch implementation between a Bazaar and a Git repository."""
173
def __init__(self, source, target):
174
super(InterToLocalGitRepository, self).__init__(source, target)
175
self.target_store = self.target.controldir._git.object_store
176
self.target_refs = self.target.controldir._git.refs
178
def _commit_needs_fetching(self, sha_id):
180
return (sha_id not in self.target_store)
181
except NoSuchRevision:
185
def _revision_needs_fetching(self, sha_id, revid):
186
if revid == NULL_REVISION:
190
sha_id = self.source_store._lookup_revision_sha1(revid)
193
return self._commit_needs_fetching(sha_id)
195
def missing_revisions(self, stop_revisions):
196
"""Find the revisions that are missing from the target repository.
198
:param stop_revisions: Revisions to check for (tuples with
200
:return: sequence of missing revisions, in topological order
201
:raise: NoSuchRevision if the stop_revisions are not present in
206
for (sha1, revid) in stop_revisions:
207
if sha1 is not None and revid is not None:
208
revid_sha_map[revid] = sha1
209
stop_revids.append(revid)
210
elif sha1 is not None:
211
if self._commit_needs_fetching(sha1):
212
for (kind, (revid, tree_sha, verifiers)) in self.source_store.lookup_git_sha(sha1):
213
revid_sha_map[revid] = sha1
214
stop_revids.append(revid)
218
stop_revids.append(revid)
220
graph = self.source.get_graph()
221
with ui.ui_factory.nested_progress_bar() as pb:
224
for revid in stop_revids:
225
sha1 = revid_sha_map.get(revid)
226
if (revid not in missing and
227
self._revision_needs_fetching(sha1, revid)):
229
new_stop_revids.append(revid)
231
parent_map = graph.get_parent_map(new_stop_revids)
232
for parent_revids in viewvalues(parent_map):
233
stop_revids.update(parent_revids)
234
pb.update("determining revisions to fetch", len(missing))
235
return graph.iter_topo_order(missing)
237
def _get_target_bzr_refs(self):
238
"""Return a dictionary with references.
240
:return: Dictionary with reference names as keys and tuples
241
with Git SHA, Bazaar revid as values.
244
for k in self.target._git.refs.allkeys():
246
v = self.target._git.refs.read_ref(k)
251
if not v.startswith(SYMREF):
253
for (kind, type_data) in self.source_store.lookup_git_sha(
255
if kind == "commit" and self.source.has_revision(
261
bzr_refs[k] = (v, revid)
264
def fetch_refs(self, update_refs, lossy, overwrite=False):
267
with self.source_store.lock_read():
268
old_refs = self._get_target_bzr_refs()
269
new_refs = update_refs(old_refs)
270
revidmap = self.fetch_objects(
271
[(git_sha, bzr_revid)
272
for (git_sha, bzr_revid) in new_refs.values()
273
if git_sha is None or not git_sha.startswith(SYMREF)],
275
for name, (gitid, revid) in viewitems(new_refs):
278
gitid = revidmap[revid][0]
280
gitid = self.source_store._lookup_revision_sha1(revid)
281
if gitid.startswith(SYMREF):
282
self.target_refs.set_symbolic_ref(
283
name, gitid[len(SYMREF):])
286
old_git_id = old_refs[name][0]
288
self.target_refs.add_if_new(name, gitid)
290
self.target_refs.set_if_equals(name, old_git_id, gitid)
291
result_refs[name] = (gitid, revid if not lossy else self.mapping.revision_id_foreign_to_bzr(gitid))
292
return revidmap, old_refs, result_refs
294
def fetch_objects(self, revs, lossy, limit=None):
295
if not lossy and not self.mapping.roundtripping:
296
for git_sha, bzr_revid in revs:
297
if (bzr_revid is not None and
298
needs_roundtripping(self.source, bzr_revid)):
299
raise NoPushSupport(self.source, self.target, self.mapping,
301
with self.source_store.lock_read():
302
todo = list(self.missing_revisions(revs))[:limit]
304
with ui.ui_factory.nested_progress_bar() as pb:
305
object_generator = MissingObjectsIterator(
306
self.source_store, self.source, pb)
307
for (old_revid, git_sha) in object_generator.import_revisions(
310
new_revid = self.mapping.revision_id_foreign_to_bzr(
313
new_revid = old_revid
315
self.mapping.revision_id_bzr_to_foreign(old_revid)
316
except InvalidRevisionId:
317
refname = self.mapping.revid_as_refname(old_revid)
318
self.target_refs[refname] = git_sha
319
revidmap[old_revid] = (git_sha, new_revid)
320
self.target_store.add_objects(object_generator)
323
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
324
fetch_spec=None, mapped_refs=None, lossy=False):
325
if mapped_refs is not None:
326
stop_revisions = mapped_refs
327
elif revision_id is not None:
328
stop_revisions = [(None, revision_id)]
329
elif fetch_spec is not None:
330
recipe = fetch_spec.get_recipe()
331
if recipe[0] in ("search", "proxy-search"):
332
stop_revisions = [(None, revid) for revid in recipe[1]]
334
raise AssertionError(
335
"Unsupported search result type %s" % recipe[0])
337
stop_revisions = [(None, revid)
338
for revid in self.source.all_revision_ids()]
341
revidmap = self.fetch_objects(stop_revisions, lossy=lossy)
342
except NoPushSupport:
343
raise NoRoundtrippingSupport(self.source, self.target)
344
return FetchResult(revidmap)
347
def is_compatible(source, target):
348
"""Be compatible with GitRepository."""
349
return (not isinstance(source, GitRepository) and
350
isinstance(target, LocalGitRepository))
353
class InterToRemoteGitRepository(InterToGitRepository):
355
def fetch_refs(self, update_refs, lossy, overwrite=False):
356
"""Import the gist of the ancestry of a particular revision."""
357
if not lossy and not self.mapping.roundtripping:
358
raise NoPushSupport(self.source, self.target, self.mapping)
359
unpeel_map = UnpeelMap.from_repository(self.source)
362
def git_update_refs(old_refs):
365
k: (v, None) for (k, v) in viewitems(old_refs)}
366
new_refs = update_refs(self.old_refs)
367
for name, (gitid, revid) in viewitems(new_refs):
369
git_sha = self.source_store._lookup_revision_sha1(revid)
370
gitid = unpeel_map.re_unpeel_tag(
371
git_sha, old_refs.get(name))
373
if remote_divergence(
374
old_refs.get(name), gitid, self.source_store):
375
raise DivergedBranches(self.source, self.target)
379
with self.source_store.lock_read():
380
new_refs = self.target.send_pack(
381
git_update_refs, self.source_store.generate_lossy_pack_data)
383
return revidmap, self.old_refs, new_refs
386
def is_compatible(source, target):
387
"""Be compatible with GitRepository."""
388
return (not isinstance(source, GitRepository) and
389
isinstance(target, RemoteGitRepository))
392
class GitSearchResult(object):
394
def __init__(self, start, exclude, keys):
396
self._exclude = exclude
402
def get_recipe(self):
403
return ('search', self._start, self._exclude, len(self._keys))
406
class InterFromGitRepository(InterRepository):
408
_matching_repo_format = GitRepositoryFormat()
410
def _target_has_shas(self, shas):
411
raise NotImplementedError(self._target_has_shas)
413
def get_determine_wants_heads(self, wants, include_tags=False, tag_selector=None):
416
def determine_wants(refs):
418
for k, v in viewitems(refs):
419
if k.endswith(ANNOTATED_TAG_SUFFIX):
420
unpeel_lookup[v] = refs[k[:-len(ANNOTATED_TAG_SUFFIX)]]
421
potential = set([unpeel_lookup.get(w, w) for w in wants])
423
for k, sha in viewitems(refs):
424
if k.endswith(ANNOTATED_TAG_SUFFIX):
427
tag_name = ref_to_tag_name(k)
430
if tag_selector and not tag_selector(tag_name):
435
return list(potential - self._target_has_shas(potential))
436
return determine_wants
438
def determine_wants_all(self, refs):
439
raise NotImplementedError(self.determine_wants_all)
442
def _get_repo_format_to_test():
445
def copy_content(self, revision_id=None):
446
"""See InterRepository.copy_content."""
447
self.fetch(revision_id, find_ghosts=False)
449
def search_missing_revision_ids(self,
450
find_ghosts=True, revision_ids=None,
451
if_present_ids=None, limit=None):
452
if limit is not None:
453
raise FetchLimitUnsupported(self)
454
if revision_ids is None and if_present_ids is None:
455
todo = set(self.source.all_revision_ids())
458
if revision_ids is not None:
459
for revid in revision_ids:
460
if not self.source.has_revision(revid):
461
raise NoSuchRevision(revid, self.source)
462
todo.update(revision_ids)
463
if if_present_ids is not None:
464
todo.update(if_present_ids)
465
result_set = todo.difference(self.target.all_revision_ids())
466
result_parents = set(itertools.chain.from_iterable(viewvalues(
467
self.source.get_graph().get_parent_map(result_set))))
468
included_keys = result_set.intersection(result_parents)
469
start_keys = result_set.difference(included_keys)
470
exclude_keys = result_parents.difference(result_set)
471
return GitSearchResult(start_keys, exclude_keys, result_set)
474
class InterGitNonGitRepository(InterFromGitRepository):
475
"""Base InterRepository that copies revisions from a Git into a non-Git
478
def _target_has_shas(self, shas):
482
revid = self.source.lookup_foreign_revision_id(sha)
483
except NotCommitError:
484
# Commit is definitely not present
488
return set([revids[r] for r in self.target.has_revisions(revids)])
490
def determine_wants_all(self, refs):
492
for k, v in viewitems(refs):
493
# For non-git target repositories, only worry about peeled
496
potential.add(self.source.controldir.get_peeled(k) or v)
497
return list(potential - self._target_has_shas(potential))
499
def _warn_slow(self):
500
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
502
'Fetching from Git to Bazaar repository. '
503
'For better performance, fetch into a Git repository.')
505
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
506
"""Fetch objects from a remote server.
508
:param determine_wants: determine_wants callback
509
:param mapping: BzrGitMapping to use
510
:param limit: Maximum number of commits to import.
511
:return: Tuple with pack hint, last imported revision id and remote
514
raise NotImplementedError(self.fetch_objects)
516
def get_determine_wants_revids(self, revids, include_tags=False, tag_selector=None):
518
for revid in set(revids):
519
if self.target.has_revision(revid):
521
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
523
return self.get_determine_wants_heads(
524
wants, include_tags=include_tags, tag_selector=tag_selector)
526
def fetch(self, revision_id=None, find_ghosts=False,
527
mapping=None, fetch_spec=None, include_tags=False, lossy=False):
529
mapping = self.source.get_mapping()
530
if revision_id is not None:
531
interesting_heads = [revision_id]
532
elif fetch_spec is not None:
533
recipe = fetch_spec.get_recipe()
534
if recipe[0] in ("search", "proxy-search"):
535
interesting_heads = recipe[1]
537
raise AssertionError("Unsupported search result type %s" %
540
interesting_heads = None
542
if interesting_heads is not None:
543
determine_wants = self.get_determine_wants_revids(
544
interesting_heads, include_tags=include_tags)
546
determine_wants = self.determine_wants_all
548
(pack_hint, _, remote_refs) = self.fetch_objects(
549
determine_wants, mapping, lossy=lossy)
550
if pack_hint is not None and self.target._format.pack_compresses:
551
self.target.pack(hint=pack_hint)
552
result = FetchResult()
553
result.refs = remote_refs
557
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
558
"""InterRepository that copies revisions from a remote Git into a non-Git
561
def get_target_heads(self):
562
# FIXME: This should be more efficient
563
all_revs = self.target.all_revision_ids()
564
parent_map = self.target.get_parent_map(all_revs)
566
for values in viewvalues(parent_map):
567
all_parents.update(values)
568
return set(all_revs) - all_parents
570
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
571
"""See `InterGitNonGitRepository`."""
573
store = get_object_store(self.target, mapping)
574
with store.lock_write():
575
heads = self.get_target_heads()
576
graph_walker = ObjectStoreGraphWalker(
577
[store._lookup_revision_sha1(head) for head in heads],
578
lambda sha: store[sha].parents)
579
wants_recorder = DetermineWantsRecorder(determine_wants)
581
with ui.ui_factory.nested_progress_bar() as pb:
582
objects_iter = self.source.fetch_objects(
583
wants_recorder, graph_walker, store.get_raw)
584
trace.mutter("Importing %d new revisions",
585
len(wants_recorder.wants))
586
(pack_hint, last_rev) = import_git_objects(
587
self.target, mapping, objects_iter, store,
588
wants_recorder.wants, pb, limit)
589
return (pack_hint, last_rev, wants_recorder.remote_refs)
592
def is_compatible(source, target):
593
"""Be compatible with GitRepository."""
594
if not isinstance(source, RemoteGitRepository):
596
if not target.supports_rich_root():
598
if isinstance(target, GitRepository):
600
if not getattr(target._format, "supports_full_versioned_files", True):
605
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
606
"""InterRepository that copies revisions from a local Git into a non-Git
609
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
610
"""See `InterGitNonGitRepository`."""
612
remote_refs = self.source.controldir.get_refs_container().as_dict()
613
wants = determine_wants(remote_refs)
614
target_git_object_retriever = get_object_store(self.target, mapping)
615
with ui.ui_factory.nested_progress_bar() as pb:
616
target_git_object_retriever.lock_write()
618
(pack_hint, last_rev) = import_git_objects(
619
self.target, mapping, self.source._git.object_store,
620
target_git_object_retriever, wants, pb, limit)
621
return (pack_hint, last_rev, remote_refs)
623
target_git_object_retriever.unlock()
626
def is_compatible(source, target):
627
"""Be compatible with GitRepository."""
628
if not isinstance(source, LocalGitRepository):
630
if not target.supports_rich_root():
632
if isinstance(target, GitRepository):
634
if not getattr(target._format, "supports_full_versioned_files", True):
639
class InterGitGitRepository(InterFromGitRepository):
640
"""InterRepository that copies between Git repositories."""
642
def fetch_refs(self, update_refs, lossy, overwrite=False):
644
raise LossyPushToSameVCS(self.source, self.target)
645
old_refs = self.target.controldir.get_refs_container()
648
def determine_wants(heads):
649
old_refs = dict([(k, (v, None))
650
for (k, v) in viewitems(heads.as_dict())])
651
new_refs = update_refs(old_refs)
652
ref_changes.update(new_refs)
653
return [sha1 for (sha1, bzr_revid) in viewvalues(new_refs)]
654
self.fetch_objects(determine_wants, lossy=lossy)
655
for k, (git_sha, bzr_revid) in viewitems(ref_changes):
656
self.target._git.refs[k] = git_sha
657
new_refs = self.target.controldir.get_refs_container()
658
return None, old_refs, new_refs
660
def fetch_objects(self, determine_wants, mapping=None, limit=None,
662
raise NotImplementedError(self.fetch_objects)
664
def _target_has_shas(self, shas):
666
[sha for sha in shas if sha in self.target._git.object_store])
668
def fetch(self, revision_id=None, find_ghosts=False,
669
mapping=None, fetch_spec=None, branches=None, limit=None,
670
include_tags=False, lossy=False):
672
mapping = self.source.get_mapping()
673
if revision_id is not None:
675
elif fetch_spec is not None:
676
recipe = fetch_spec.get_recipe()
677
if recipe[0] in ("search", "proxy-search"):
680
raise AssertionError(
681
"Unsupported search result type %s" % recipe[0])
683
if branches is not None:
684
determine_wants = self.get_determine_wants_branches(
685
branches, include_tags=include_tags)
686
elif fetch_spec is None and revision_id is None:
687
determine_wants = self.determine_wants_all
689
determine_wants = self.get_determine_wants_revids(
690
args, include_tags=include_tags)
691
wants_recorder = DetermineWantsRecorder(determine_wants)
692
self.fetch_objects(wants_recorder, mapping, limit=limit, lossy=lossy)
693
result = FetchResult()
694
result.refs = wants_recorder.remote_refs
697
def get_determine_wants_revids(self, revids, include_tags=False, tag_selector=None):
699
for revid in set(revids):
700
if revid == NULL_REVISION:
702
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
704
return self.get_determine_wants_heads(wants, include_tags=include_tags, tag_selector=tag_selector)
706
def get_determine_wants_branches(self, branches, include_tags=False):
707
def determine_wants(refs):
709
for name, value in viewitems(refs):
710
if value == ZERO_SHA:
713
if name.endswith(ANNOTATED_TAG_SUFFIX):
716
if name in branches or (include_tags and is_tag(name)):
719
return determine_wants
721
def determine_wants_all(self, refs):
723
v for k, v in refs.items()
724
if not v == ZERO_SHA and not k.endswith(ANNOTATED_TAG_SUFFIX)])
725
return list(potential - self._target_has_shas(potential))
728
class InterLocalGitLocalGitRepository(InterGitGitRepository):
730
def fetch_objects(self, determine_wants, mapping=None, limit=None,
733
raise LossyPushToSameVCS(self.source, self.target)
734
if limit is not None:
735
raise FetchLimitUnsupported(self)
736
from .remote import DefaultProgressReporter
737
with ui.ui_factory.nested_progress_bar() as pb:
738
progress = DefaultProgressReporter(pb).progress
739
refs = self.source._git.fetch(
740
self.target._git, determine_wants,
742
return (None, None, refs)
745
def is_compatible(source, target):
746
"""Be compatible with GitRepository."""
747
return (isinstance(source, LocalGitRepository) and
748
isinstance(target, LocalGitRepository))
751
class InterRemoteGitLocalGitRepository(InterGitGitRepository):
753
def fetch_objects(self, determine_wants, mapping=None, limit=None,
756
raise LossyPushToSameVCS(self.source, self.target)
757
if limit is not None:
758
raise FetchLimitUnsupported(self)
759
graphwalker = self.target._git.get_graph_walker()
760
if (CAPABILITY_THIN_PACK in
761
self.source.controldir._client._fetch_capabilities):
762
# TODO(jelmer): Avoid reading entire file into memory and
763
# only processing it after the whole file has been fetched.
769
self.target._git.object_store.move_in_thin_pack(f)
774
f, commit, abort = self.target._git.object_store.add_pack()
776
refs = self.source.controldir.fetch_pack(
777
determine_wants, graphwalker, f.write)
779
return (None, None, refs)
780
except BaseException:
785
def is_compatible(source, target):
786
"""Be compatible with GitRepository."""
787
return (isinstance(source, RemoteGitRepository) and
788
isinstance(target, LocalGitRepository))
added
removed
breezy/git/interrepo.py
