1
# Copyright (C) 2009-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""InterRepository operations."""
19
from io import BytesIO
22
from dulwich.errors import (
25
from dulwich.object_store import (
26
ObjectStoreGraphWalker,
28
from dulwich.protocol import (
32
from dulwich.refs import (
36
from dulwich.walk import Walker
38
from ..errors import (
40
FetchLimitUnsupported,
43
NoRoundtrippingSupport,
46
from ..repository import (
50
from ..revision import (
64
DetermineWantsRecorder,
66
from .mapping import (
69
from .object_store import (
73
MissingObjectsIterator,
79
from .repository import (
87
from .unpeel_map import (
92
class InterToGitRepository(InterRepository):
93
"""InterRepository that copies into a Git repository."""
95
_matching_repo_format = GitRepositoryFormat()
97
def __init__(self, source, target):
98
super(InterToGitRepository, self).__init__(source, target)
99
self.mapping = self.target.get_mapping()
100
self.source_store = get_object_store(self.source, self.mapping)
103
def _get_repo_format_to_test():
106
def copy_content(self, revision_id=None, pb=None):
107
"""See InterRepository.copy_content."""
108
self.fetch(revision_id, pb, find_ghosts=False)
110
def fetch_refs(self, update_refs, lossy, overwrite=False):
111
"""Fetch possibly roundtripped revisions into the target repository
114
:param update_refs: Generate refs to fetch. Receives dictionary
115
with old refs (git shas), returns dictionary of new names to
117
:param lossy: Whether to roundtrip
118
:return: old refs, new refs
120
raise NotImplementedError(self.fetch_refs)
122
def search_missing_revision_ids(self,
123
find_ghosts=True, revision_ids=None,
124
if_present_ids=None, limit=None):
125
if limit is not None:
126
raise FetchLimitUnsupported(self)
130
todo.extend(revision_ids)
132
todo.extend(revision_ids)
133
with self.source_store.lock_read():
134
for revid in revision_ids:
135
if revid == NULL_REVISION:
138
git_sha = self.source_store._lookup_revision_sha1(revid)
140
raise NoSuchRevision(revid, self.source)
141
git_shas.append(git_sha)
146
sha for sha in self.target.controldir.get_refs_container().as_dict().values()
148
missing_revids = set()
150
for (kind, type_data) in self.source_store.lookup_git_sha(
153
missing_revids.add(type_data[0])
154
return self.source.revision_ids_to_search_result(missing_revids)
156
def _warn_slow(self):
157
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
159
'Pushing from a Bazaar to a Git repository. '
160
'For better performance, push into a Bazaar repository.')
163
class InterToLocalGitRepository(InterToGitRepository):
164
"""InterBranch implementation between a Bazaar and a Git repository."""
166
def __init__(self, source, target):
167
super(InterToLocalGitRepository, self).__init__(source, target)
168
self.target_store = self.target.controldir._git.object_store
169
self.target_refs = self.target.controldir._git.refs
171
def _commit_needs_fetching(self, sha_id):
173
return (sha_id not in self.target_store)
174
except NoSuchRevision:
178
def _revision_needs_fetching(self, sha_id, revid):
179
if revid == NULL_REVISION:
183
sha_id = self.source_store._lookup_revision_sha1(revid)
186
return self._commit_needs_fetching(sha_id)
188
def missing_revisions(self, stop_revisions):
189
"""Find the revisions that are missing from the target repository.
191
:param stop_revisions: Revisions to check for (tuples with
193
:return: sequence of missing revisions, in topological order
194
:raise: NoSuchRevision if the stop_revisions are not present in
199
for (sha1, revid) in stop_revisions:
200
if sha1 is not None and revid is not None:
201
revid_sha_map[revid] = sha1
202
stop_revids.append(revid)
203
elif sha1 is not None:
204
if self._commit_needs_fetching(sha1):
205
for (kind, (revid, tree_sha, verifiers)) in self.source_store.lookup_git_sha(sha1):
206
revid_sha_map[revid] = sha1
207
stop_revids.append(revid)
211
stop_revids.append(revid)
213
graph = self.source.get_graph()
214
with ui.ui_factory.nested_progress_bar() as pb:
217
for revid in stop_revids:
218
sha1 = revid_sha_map.get(revid)
219
if (revid not in missing and
220
self._revision_needs_fetching(sha1, revid)):
222
new_stop_revids.append(revid)
224
parent_map = graph.get_parent_map(new_stop_revids)
225
for parent_revids in parent_map.values():
226
stop_revids.update(parent_revids)
227
pb.update("determining revisions to fetch", len(missing))
228
return graph.iter_topo_order(missing)
230
def _get_target_bzr_refs(self):
231
"""Return a dictionary with references.
233
:return: Dictionary with reference names as keys and tuples
234
with Git SHA, Bazaar revid as values.
237
for k in self.target._git.refs.allkeys():
239
v = self.target._git.refs.read_ref(k)
244
if not v.startswith(SYMREF):
246
for (kind, type_data) in self.source_store.lookup_git_sha(
248
if kind == "commit" and self.source.has_revision(
254
bzr_refs[k] = (v, revid)
257
def fetch_refs(self, update_refs, lossy, overwrite=False):
260
with self.source_store.lock_read():
261
old_refs = self._get_target_bzr_refs()
262
new_refs = update_refs(old_refs)
263
revidmap = self.fetch_objects(
264
[(git_sha, bzr_revid)
265
for (git_sha, bzr_revid) in new_refs.values()
266
if git_sha is None or not git_sha.startswith(SYMREF)],
268
for name, (gitid, revid) in new_refs.items():
271
gitid = revidmap[revid][0]
273
gitid = self.source_store._lookup_revision_sha1(revid)
274
if gitid.startswith(SYMREF):
275
self.target_refs.set_symbolic_ref(
276
name, gitid[len(SYMREF):])
279
old_git_id = old_refs[name][0]
281
self.target_refs.add_if_new(name, gitid)
283
self.target_refs.set_if_equals(name, old_git_id, gitid)
284
result_refs[name] = (gitid, revid if not lossy else self.mapping.revision_id_foreign_to_bzr(gitid))
285
return revidmap, old_refs, result_refs
287
def fetch_objects(self, revs, lossy, limit=None):
288
if not lossy and not self.mapping.roundtripping:
289
for git_sha, bzr_revid in revs:
290
if (bzr_revid is not None and
291
needs_roundtripping(self.source, bzr_revid)):
292
raise NoPushSupport(self.source, self.target, self.mapping,
294
with self.source_store.lock_read():
295
todo = list(self.missing_revisions(revs))[:limit]
297
with ui.ui_factory.nested_progress_bar() as pb:
298
object_generator = MissingObjectsIterator(
299
self.source_store, self.source, pb)
300
for (old_revid, git_sha) in object_generator.import_revisions(
303
new_revid = self.mapping.revision_id_foreign_to_bzr(
306
new_revid = old_revid
308
self.mapping.revision_id_bzr_to_foreign(old_revid)
309
except InvalidRevisionId:
310
refname = self.mapping.revid_as_refname(old_revid)
311
self.target_refs[refname] = git_sha
312
revidmap[old_revid] = (git_sha, new_revid)
313
self.target_store.add_objects(object_generator)
316
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
317
fetch_spec=None, mapped_refs=None, lossy=False):
318
if mapped_refs is not None:
319
stop_revisions = mapped_refs
320
elif revision_id is not None:
321
stop_revisions = [(None, revision_id)]
322
elif fetch_spec is not None:
323
recipe = fetch_spec.get_recipe()
324
if recipe[0] in ("search", "proxy-search"):
325
stop_revisions = [(None, revid) for revid in recipe[1]]
327
raise AssertionError(
328
"Unsupported search result type %s" % recipe[0])
330
stop_revisions = [(None, revid)
331
for revid in self.source.all_revision_ids()]
334
revidmap = self.fetch_objects(stop_revisions, lossy=lossy)
335
except NoPushSupport:
336
raise NoRoundtrippingSupport(self.source, self.target)
337
return FetchResult(revidmap)
340
def is_compatible(source, target):
341
"""Be compatible with GitRepository."""
342
return (not isinstance(source, GitRepository) and
343
isinstance(target, LocalGitRepository))
346
class InterToRemoteGitRepository(InterToGitRepository):
348
def fetch_refs(self, update_refs, lossy, overwrite=False):
349
"""Import the gist of the ancestry of a particular revision."""
350
if not lossy and not self.mapping.roundtripping:
351
raise NoPushSupport(self.source, self.target, self.mapping)
352
unpeel_map = UnpeelMap.from_repository(self.source)
355
def git_update_refs(old_refs):
358
k: (v, None) for (k, v) in old_refs.items()}
359
new_refs = update_refs(self.old_refs)
360
for name, (gitid, revid) in new_refs.items():
362
git_sha = self.source_store._lookup_revision_sha1(revid)
363
gitid = unpeel_map.re_unpeel_tag(
364
git_sha, old_refs.get(name))
366
if remote_divergence(
367
old_refs.get(name), gitid, self.source_store):
368
raise DivergedBranches(self.source, self.target)
372
with self.source_store.lock_read():
373
new_refs = self.target.send_pack(
374
git_update_refs, self.source_store.generate_lossy_pack_data)
376
return revidmap, self.old_refs, new_refs
379
def is_compatible(source, target):
380
"""Be compatible with GitRepository."""
381
return (not isinstance(source, GitRepository) and
382
isinstance(target, RemoteGitRepository))
385
class GitSearchResult(object):
387
def __init__(self, start, exclude, keys):
389
self._exclude = exclude
395
def get_recipe(self):
396
return ('search', self._start, self._exclude, len(self._keys))
399
class InterFromGitRepository(InterRepository):
401
_matching_repo_format = GitRepositoryFormat()
403
def _target_has_shas(self, shas):
404
raise NotImplementedError(self._target_has_shas)
406
def get_determine_wants_heads(self, wants, include_tags=False):
409
def determine_wants(refs):
411
for k, v in refs.items():
412
if k.endswith(ANNOTATED_TAG_SUFFIX):
413
unpeel_lookup[v] = refs[k[:-len(ANNOTATED_TAG_SUFFIX)]]
414
potential = set([unpeel_lookup.get(w, w) for w in wants])
416
for k, sha in refs.items():
417
if k.endswith(ANNOTATED_TAG_SUFFIX):
424
return list(potential - self._target_has_shas(potential))
425
return determine_wants
427
def determine_wants_all(self, refs):
428
raise NotImplementedError(self.determine_wants_all)
431
def _get_repo_format_to_test():
434
def copy_content(self, revision_id=None):
435
"""See InterRepository.copy_content."""
436
self.fetch(revision_id, find_ghosts=False)
438
def search_missing_revision_ids(self,
439
find_ghosts=True, revision_ids=None,
440
if_present_ids=None, limit=None):
441
if limit is not None:
442
raise FetchLimitUnsupported(self)
443
if revision_ids is None and if_present_ids is None:
444
todo = set(self.source.all_revision_ids())
447
if revision_ids is not None:
448
for revid in revision_ids:
449
if not self.source.has_revision(revid):
450
raise NoSuchRevision(revid, self.source)
451
todo.update(revision_ids)
452
if if_present_ids is not None:
453
todo.update(if_present_ids)
454
result_set = todo.difference(self.target.all_revision_ids())
455
result_parents = set(itertools.chain.from_iterable(
456
self.source.get_graph().get_parent_map(result_set).values()))
457
included_keys = result_set.intersection(result_parents)
458
start_keys = result_set.difference(included_keys)
459
exclude_keys = result_parents.difference(result_set)
460
return GitSearchResult(start_keys, exclude_keys, result_set)
463
class InterGitNonGitRepository(InterFromGitRepository):
464
"""Base InterRepository that copies revisions from a Git into a non-Git
467
def _target_has_shas(self, shas):
471
revid = self.source.lookup_foreign_revision_id(sha)
472
except NotCommitError:
473
# Commit is definitely not present
477
return set([revids[r] for r in self.target.has_revisions(revids)])
479
def determine_wants_all(self, refs):
481
for k, v in refs.items():
482
# For non-git target repositories, only worry about peeled
485
potential.add(self.source.controldir.get_peeled(k) or v)
486
return list(potential - self._target_has_shas(potential))
488
def _warn_slow(self):
489
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
491
'Fetching from Git to Bazaar repository. '
492
'For better performance, fetch into a Git repository.')
494
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
495
"""Fetch objects from a remote server.
497
:param determine_wants: determine_wants callback
498
:param mapping: BzrGitMapping to use
499
:param limit: Maximum number of commits to import.
500
:return: Tuple with pack hint, last imported revision id and remote
503
raise NotImplementedError(self.fetch_objects)
505
def get_determine_wants_revids(self, revids, include_tags=False):
507
for revid in set(revids):
508
if self.target.has_revision(revid):
510
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
512
return self.get_determine_wants_heads(wants, include_tags=include_tags)
514
def fetch(self, revision_id=None, find_ghosts=False,
515
mapping=None, fetch_spec=None, include_tags=False, lossy=False):
517
mapping = self.source.get_mapping()
518
if revision_id is not None:
519
interesting_heads = [revision_id]
520
elif fetch_spec is not None:
521
recipe = fetch_spec.get_recipe()
522
if recipe[0] in ("search", "proxy-search"):
523
interesting_heads = recipe[1]
525
raise AssertionError("Unsupported search result type %s" %
528
interesting_heads = None
530
if interesting_heads is not None:
531
determine_wants = self.get_determine_wants_revids(
532
interesting_heads, include_tags=include_tags)
534
determine_wants = self.determine_wants_all
536
(pack_hint, _, remote_refs) = self.fetch_objects(
537
determine_wants, mapping, lossy=lossy)
538
if pack_hint is not None and self.target._format.pack_compresses:
539
self.target.pack(hint=pack_hint)
540
result = FetchResult()
541
result.refs = remote_refs
545
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
546
"""InterRepository that copies revisions from a remote Git into a non-Git
549
def get_target_heads(self):
550
# FIXME: This should be more efficient
551
all_revs = self.target.all_revision_ids()
552
parent_map = self.target.get_parent_map(all_revs)
554
for values in parent_map.values():
555
all_parents.update(values)
556
return set(all_revs) - all_parents
558
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
559
"""See `InterGitNonGitRepository`."""
561
store = get_object_store(self.target, mapping)
562
with store.lock_write():
563
heads = self.get_target_heads()
564
graph_walker = ObjectStoreGraphWalker(
565
[store._lookup_revision_sha1(head) for head in heads],
566
lambda sha: store[sha].parents)
567
wants_recorder = DetermineWantsRecorder(determine_wants)
569
with ui.ui_factory.nested_progress_bar() as pb:
570
objects_iter = self.source.fetch_objects(
571
wants_recorder, graph_walker, store.get_raw)
572
trace.mutter("Importing %d new revisions",
573
len(wants_recorder.wants))
574
(pack_hint, last_rev) = import_git_objects(
575
self.target, mapping, objects_iter, store,
576
wants_recorder.wants, pb, limit)
577
return (pack_hint, last_rev, wants_recorder.remote_refs)
580
def is_compatible(source, target):
581
"""Be compatible with GitRepository."""
582
if not isinstance(source, RemoteGitRepository):
584
if not target.supports_rich_root():
586
if isinstance(target, GitRepository):
588
if not getattr(target._format, "supports_full_versioned_files", True):
593
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
594
"""InterRepository that copies revisions from a local Git into a non-Git
597
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
598
"""See `InterGitNonGitRepository`."""
600
remote_refs = self.source.controldir.get_refs_container().as_dict()
601
wants = determine_wants(remote_refs)
602
target_git_object_retriever = get_object_store(self.target, mapping)
603
with ui.ui_factory.nested_progress_bar() as pb:
604
target_git_object_retriever.lock_write()
606
(pack_hint, last_rev) = import_git_objects(
607
self.target, mapping, self.source._git.object_store,
608
target_git_object_retriever, wants, pb, limit)
609
return (pack_hint, last_rev, remote_refs)
611
target_git_object_retriever.unlock()
614
def is_compatible(source, target):
615
"""Be compatible with GitRepository."""
616
if not isinstance(source, LocalGitRepository):
618
if not target.supports_rich_root():
620
if isinstance(target, GitRepository):
622
if not getattr(target._format, "supports_full_versioned_files", True):
627
class InterGitGitRepository(InterFromGitRepository):
628
"""InterRepository that copies between Git repositories."""
630
def fetch_refs(self, update_refs, lossy, overwrite=False):
632
raise LossyPushToSameVCS(self.source, self.target)
633
old_refs = self.target.controldir.get_refs_container()
636
def determine_wants(heads):
637
old_refs = dict([(k, (v, None))
638
for (k, v) in heads.as_dict().items()])
639
new_refs = update_refs(old_refs)
640
ref_changes.update(new_refs)
641
return [sha1 for (sha1, bzr_revid) in new_refs.values()]
642
self.fetch_objects(determine_wants, lossy=lossy)
643
for k, (git_sha, bzr_revid) in ref_changes.items():
644
self.target._git.refs[k] = git_sha
645
new_refs = self.target.controldir.get_refs_container()
646
return None, old_refs, new_refs
648
def fetch_objects(self, determine_wants, mapping=None, limit=None,
650
raise NotImplementedError(self.fetch_objects)
652
def _target_has_shas(self, shas):
654
[sha for sha in shas if sha in self.target._git.object_store])
656
def fetch(self, revision_id=None, find_ghosts=False,
657
mapping=None, fetch_spec=None, branches=None, limit=None,
658
include_tags=False, lossy=False):
660
mapping = self.source.get_mapping()
661
if revision_id is not None:
663
elif fetch_spec is not None:
664
recipe = fetch_spec.get_recipe()
665
if recipe[0] in ("search", "proxy-search"):
668
raise AssertionError(
669
"Unsupported search result type %s" % recipe[0])
671
if branches is not None:
672
determine_wants = self.get_determine_wants_branches(
673
branches, include_tags=include_tags)
674
elif fetch_spec is None and revision_id is None:
675
determine_wants = self.determine_wants_all
677
determine_wants = self.get_determine_wants_revids(
678
args, include_tags=include_tags)
679
wants_recorder = DetermineWantsRecorder(determine_wants)
680
self.fetch_objects(wants_recorder, mapping, limit=limit, lossy=lossy)
681
result = FetchResult()
682
result.refs = wants_recorder.remote_refs
685
def get_determine_wants_revids(self, revids, include_tags=False):
687
for revid in set(revids):
688
if revid == NULL_REVISION:
690
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
692
return self.get_determine_wants_heads(wants, include_tags=include_tags)
694
def get_determine_wants_branches(self, branches, include_tags=False):
695
def determine_wants(refs):
697
for name, value in refs.items():
698
if value == ZERO_SHA:
701
if name.endswith(ANNOTATED_TAG_SUFFIX):
704
if name in branches or (include_tags and is_tag(name)):
707
return determine_wants
709
def determine_wants_all(self, refs):
711
v for k, v in refs.items()
712
if not v == ZERO_SHA and not k.endswith(ANNOTATED_TAG_SUFFIX)])
713
return list(potential - self._target_has_shas(potential))
716
class InterLocalGitLocalGitRepository(InterGitGitRepository):
718
def fetch_objects(self, determine_wants, mapping=None, limit=None,
721
raise LossyPushToSameVCS(self.source, self.target)
722
if limit is not None:
723
raise FetchLimitUnsupported(self)
724
from .remote import DefaultProgressReporter
725
with ui.ui_factory.nested_progress_bar() as pb:
726
progress = DefaultProgressReporter(pb).progress
727
refs = self.source._git.fetch(
728
self.target._git, determine_wants,
730
return (None, None, refs)
733
def is_compatible(source, target):
734
"""Be compatible with GitRepository."""
735
return (isinstance(source, LocalGitRepository) and
736
isinstance(target, LocalGitRepository))
739
class InterRemoteGitLocalGitRepository(InterGitGitRepository):
741
def fetch_objects(self, determine_wants, mapping=None, limit=None,
744
raise LossyPushToSameVCS(self.source, self.target)
745
if limit is not None:
746
raise FetchLimitUnsupported(self)
747
graphwalker = self.target._git.get_graph_walker()
748
if (CAPABILITY_THIN_PACK in
749
self.source.controldir._client._fetch_capabilities):
750
# TODO(jelmer): Avoid reading entire file into memory and
751
# only processing it after the whole file has been fetched.
757
self.target._git.object_store.move_in_thin_pack(f)
762
f, commit, abort = self.target._git.object_store.add_pack()
764
refs = self.source.controldir.fetch_pack(
765
determine_wants, graphwalker, f.write)
767
return (None, None, refs)
768
except BaseException:
773
def is_compatible(source, target):
774
"""Be compatible with GitRepository."""
775
return (isinstance(source, RemoteGitRepository) and
776
isinstance(target, LocalGitRepository))
added
removed
breezy/git/interrepo.py
