1
# Copyright (C) 2009-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""InterRepository operations."""
19
from __future__ import absolute_import
21
from io import BytesIO
24
from dulwich.errors import (
27
from dulwich.object_store import (
28
ObjectStoreGraphWalker,
30
from dulwich.protocol import (
34
from dulwich.refs import (
38
from dulwich.walk import Walker
40
from ..errors import (
42
FetchLimitUnsupported,
45
NoRoundtrippingSupport,
48
from ..repository import (
51
from ..revision import (
54
from ..sixish import (
69
DetermineWantsRecorder,
71
from .mapping import (
74
from .object_store import (
78
MissingObjectsIterator,
84
from .repository import (
92
from .unpeel_map import (
97
class InterToGitRepository(InterRepository):
98
"""InterRepository that copies into a Git repository."""
100
_matching_repo_format = GitRepositoryFormat()
102
def __init__(self, source, target):
103
super(InterToGitRepository, self).__init__(source, target)
104
self.mapping = self.target.get_mapping()
105
self.source_store = get_object_store(self.source, self.mapping)
108
def _get_repo_format_to_test():
111
def copy_content(self, revision_id=None, pb=None):
112
"""See InterRepository.copy_content."""
113
self.fetch(revision_id, pb, find_ghosts=False)
115
def fetch_refs(self, update_refs, lossy, overwrite=False):
116
"""Fetch possibly roundtripped revisions into the target repository
119
:param update_refs: Generate refs to fetch. Receives dictionary
120
with old refs (git shas), returns dictionary of new names to
122
:param lossy: Whether to roundtrip
123
:return: old refs, new refs
125
raise NotImplementedError(self.fetch_refs)
127
def search_missing_revision_ids(self,
128
find_ghosts=True, revision_ids=None, if_present_ids=None,
130
if limit is not None:
131
raise FetchLimitUnsupported(self)
135
todo.extend(revision_ids)
137
todo.extend(revision_ids)
138
with self.source_store.lock_read():
139
for revid in revision_ids:
140
if revid == NULL_REVISION:
143
git_sha = self.source_store._lookup_revision_sha1(revid)
145
raise NoSuchRevision(revid, self.source)
146
git_shas.append(git_sha)
147
walker = Walker(self.source_store,
148
include=git_shas, exclude=[
149
sha for sha in self.target.controldir.get_refs_container().as_dict().values()
151
missing_revids = set()
153
for (kind, type_data) in self.source_store.lookup_git_sha(entry.commit.id):
155
missing_revids.add(type_data[0])
156
return self.source.revision_ids_to_search_result(missing_revids)
158
def _warn_slow(self):
159
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
161
'Pushing from a Bazaar to a Git repository. '
162
'For better performance, push into a Bazaar repository.')
165
class InterToLocalGitRepository(InterToGitRepository):
166
"""InterBranch implementation between a Bazaar and a Git repository."""
168
def __init__(self, source, target):
169
super(InterToLocalGitRepository, self).__init__(source, target)
170
self.target_store = self.target.controldir._git.object_store
171
self.target_refs = self.target.controldir._git.refs
173
def _commit_needs_fetching(self, sha_id):
175
return (sha_id not in self.target_store)
176
except NoSuchRevision:
180
def _revision_needs_fetching(self, sha_id, revid):
181
if revid == NULL_REVISION:
185
sha_id = self.source_store._lookup_revision_sha1(revid)
188
return self._commit_needs_fetching(sha_id)
190
def missing_revisions(self, stop_revisions):
191
"""Find the revisions that are missing from the target repository.
193
:param stop_revisions: Revisions to check for (tuples with
195
:return: sequence of missing revisions, in topological order
196
:raise: NoSuchRevision if the stop_revisions are not present in
201
for (sha1, revid) in stop_revisions:
202
if sha1 is not None and revid is not None:
203
revid_sha_map[revid] = sha1
204
stop_revids.append(revid)
205
elif sha1 is not None:
206
if self._commit_needs_fetching(sha1):
207
for (kind, (revid, tree_sha, verifiers)) in self.source_store.lookup_git_sha(sha1):
208
revid_sha_map[revid] = sha1
209
stop_revids.append(revid)
213
stop_revids.append(revid)
215
graph = self.source.get_graph()
216
pb = ui.ui_factory.nested_progress_bar()
220
for revid in stop_revids:
221
sha1 = revid_sha_map.get(revid)
222
if (not revid in missing and
223
self._revision_needs_fetching(sha1, revid)):
225
new_stop_revids.append(revid)
227
parent_map = graph.get_parent_map(new_stop_revids)
228
for parent_revids in viewvalues(parent_map):
229
stop_revids.update(parent_revids)
230
pb.update("determining revisions to fetch", len(missing))
233
return graph.iter_topo_order(missing)
235
def _get_target_bzr_refs(self):
236
"""Return a dictionary with references.
238
:return: Dictionary with reference names as keys and tuples
239
with Git SHA, Bazaar revid as values.
243
for k in self.target._git.refs.allkeys():
245
v = self.target._git.refs.read_ref(k)
250
if not v.startswith(SYMREF):
252
for (kind, type_data) in self.source_store.lookup_git_sha(v):
253
if kind == "commit" and self.source.has_revision(type_data[0]):
258
bzr_refs[k] = (v, revid)
261
def fetch_refs(self, update_refs, lossy, overwrite=False):
263
with self.source_store.lock_read():
264
old_refs = self._get_target_bzr_refs()
265
new_refs = update_refs(old_refs)
266
revidmap = self.fetch_objects(
267
[(git_sha, bzr_revid) for (git_sha, bzr_revid) in new_refs.values() if git_sha is None or not git_sha.startswith(SYMREF)], lossy=lossy)
268
for name, (gitid, revid) in viewitems(new_refs):
271
gitid = revidmap[revid][0]
273
gitid = self.source_store._lookup_revision_sha1(revid)
274
if gitid.startswith(SYMREF):
275
self.target_refs.set_symbolic_ref(name, gitid[len(SYMREF):])
278
old_git_id = old_refs[name][0]
280
self.target_refs.add_if_new(name, gitid)
282
self.target_refs.set_if_equals(name, old_git_id, gitid)
283
return revidmap, old_refs, new_refs
285
def fetch_objects(self, revs, lossy, limit=None):
286
if not lossy and not self.mapping.roundtripping:
287
for git_sha, bzr_revid in revs:
288
if bzr_revid is not None and needs_roundtripping(self.source, bzr_revid):
289
raise NoPushSupport(self.source, self.target, self.mapping,
291
with self.source_store.lock_read():
292
todo = list(self.missing_revisions(revs))[:limit]
294
pb = ui.ui_factory.nested_progress_bar()
296
object_generator = MissingObjectsIterator(
297
self.source_store, self.source, pb)
298
for (old_revid, git_sha) in object_generator.import_revisions(
301
new_revid = self.mapping.revision_id_foreign_to_bzr(git_sha)
303
new_revid = old_revid
305
self.mapping.revision_id_bzr_to_foreign(old_revid)
306
except InvalidRevisionId:
307
refname = self.mapping.revid_as_refname(old_revid)
308
self.target_refs[refname] = git_sha
309
revidmap[old_revid] = (git_sha, new_revid)
310
self.target_store.add_objects(object_generator)
315
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
316
fetch_spec=None, mapped_refs=None):
317
if mapped_refs is not None:
318
stop_revisions = mapped_refs
319
elif revision_id is not None:
320
stop_revisions = [(None, revision_id)]
321
elif fetch_spec is not None:
322
recipe = fetch_spec.get_recipe()
323
if recipe[0] in ("search", "proxy-search"):
324
stop_revisions = [(None, revid) for revid in recipe[1]]
326
raise AssertionError("Unsupported search result type %s" % recipe[0])
328
stop_revisions = [(None, revid) for revid in self.source.all_revision_ids()]
331
self.fetch_objects(stop_revisions, lossy=False)
332
except NoPushSupport:
333
raise NoRoundtrippingSupport(self.source, self.target)
336
def is_compatible(source, target):
337
"""Be compatible with GitRepository."""
338
return (not isinstance(source, GitRepository) and
339
isinstance(target, LocalGitRepository))
342
class InterToRemoteGitRepository(InterToGitRepository):
344
def fetch_refs(self, update_refs, lossy, overwrite=False):
345
"""Import the gist of the ancestry of a particular revision."""
346
if not lossy and not self.mapping.roundtripping:
347
raise NoPushSupport(self.source, self.target, self.mapping)
348
unpeel_map = UnpeelMap.from_repository(self.source)
350
def git_update_refs(old_refs):
352
self.old_refs = dict([(k, (v, None)) for (k, v) in viewitems(old_refs)])
353
self.new_refs = update_refs(self.old_refs)
354
for name, (gitid, revid) in viewitems(self.new_refs):
356
git_sha = self.source_store._lookup_revision_sha1(revid)
357
gitid = unpeel_map.re_unpeel_tag(git_sha, old_refs.get(name))
359
if remote_divergence(old_refs.get(name), gitid, self.source_store):
360
raise DivergedBranches(self.source, self.target)
364
with self.source_store.lock_read():
365
new_refs = self.target.send_pack(git_update_refs,
366
self.source_store.generate_lossy_pack_data)
368
return revidmap, self.old_refs, self.new_refs
371
def is_compatible(source, target):
372
"""Be compatible with GitRepository."""
373
return (not isinstance(source, GitRepository) and
374
isinstance(target, RemoteGitRepository))
377
class GitSearchResult(object):
379
def __init__(self, start, exclude, keys):
381
self._exclude = exclude
387
def get_recipe(self):
388
return ('search', self._start, self._exclude, len(self._keys))
391
class InterFromGitRepository(InterRepository):
393
_matching_repo_format = GitRepositoryFormat()
395
def _target_has_shas(self, shas):
396
raise NotImplementedError(self._target_has_shas)
398
def get_determine_wants_heads(self, wants, include_tags=False):
400
def determine_wants(refs):
402
for k, v in viewitems(refs):
403
if k.endswith(ANNOTATED_TAG_SUFFIX):
404
unpeel_lookup[v] = refs[k[:-len(ANNOTATED_TAG_SUFFIX)]]
405
potential = set([unpeel_lookup.get(w, w) for w in wants])
407
for k, sha in viewitems(refs):
408
if k.endswith(ANNOTATED_TAG_SUFFIX):
415
return list(potential - self._target_has_shas(potential))
416
return determine_wants
418
def determine_wants_all(self, refs):
419
raise NotImplementedError(self.determine_wants_all)
422
def _get_repo_format_to_test():
425
def copy_content(self, revision_id=None):
426
"""See InterRepository.copy_content."""
427
self.fetch(revision_id, find_ghosts=False)
429
def search_missing_revision_ids(self,
430
find_ghosts=True, revision_ids=None, if_present_ids=None,
432
if limit is not None:
433
raise FetchLimitUnsupported(self)
434
if revision_ids is None and if_present_ids is None:
435
todo = set(self.source.all_revision_ids())
438
if revision_ids is not None:
439
for revid in revision_ids:
440
if not self.source.has_revision(revid):
441
raise NoSuchRevision(revid, self.source)
442
todo.update(revision_ids)
443
if if_present_ids is not None:
444
todo.update(if_present_ids)
445
result_set = todo.difference(self.target.all_revision_ids())
446
result_parents = set(itertools.chain.from_iterable(viewvalues(
447
self.source.get_graph().get_parent_map(result_set))))
448
included_keys = result_set.intersection(result_parents)
449
start_keys = result_set.difference(included_keys)
450
exclude_keys = result_parents.difference(result_set)
451
return GitSearchResult(start_keys, exclude_keys, result_set)
454
class InterGitNonGitRepository(InterFromGitRepository):
455
"""Base InterRepository that copies revisions from a Git into a non-Git
458
def _target_has_shas(self, shas):
462
revid = self.source.lookup_foreign_revision_id(sha)
463
except NotCommitError:
464
# Commit is definitely not present
468
return set([revids[r] for r in self.target.has_revisions(revids)])
470
def determine_wants_all(self, refs):
472
for k, v in viewitems(refs):
473
# For non-git target repositories, only worry about peeled
476
potential.add(self.source.controldir.get_peeled(k) or v)
477
return list(potential - self._target_has_shas(potential))
479
def _warn_slow(self):
480
if not config.GlobalConfig().suppress_warning('slow_intervcs_push'):
482
'Fetching from Git to Bazaar repository. '
483
'For better performance, fetch into a Git repository.')
485
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
486
"""Fetch objects from a remote server.
488
:param determine_wants: determine_wants callback
489
:param mapping: BzrGitMapping to use
490
:param limit: Maximum number of commits to import.
491
:return: Tuple with pack hint, last imported revision id and remote refs
493
raise NotImplementedError(self.fetch_objects)
495
def get_determine_wants_revids(self, revids, include_tags=False):
497
for revid in set(revids):
498
if self.target.has_revision(revid):
500
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
502
return self.get_determine_wants_heads(wants, include_tags=include_tags)
504
def fetch(self, revision_id=None, find_ghosts=False,
505
mapping=None, fetch_spec=None, include_tags=False):
507
mapping = self.source.get_mapping()
508
if revision_id is not None:
509
interesting_heads = [revision_id]
510
elif fetch_spec is not None:
511
recipe = fetch_spec.get_recipe()
512
if recipe[0] in ("search", "proxy-search"):
513
interesting_heads = recipe[1]
515
raise AssertionError("Unsupported search result type %s" %
518
interesting_heads = None
520
if interesting_heads is not None:
521
determine_wants = self.get_determine_wants_revids(
522
interesting_heads, include_tags=include_tags)
524
determine_wants = self.determine_wants_all
526
(pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
528
if pack_hint is not None and self.target._format.pack_compresses:
529
self.target.pack(hint=pack_hint)
533
class InterRemoteGitNonGitRepository(InterGitNonGitRepository):
534
"""InterRepository that copies revisions from a remote Git into a non-Git
537
def get_target_heads(self):
538
# FIXME: This should be more efficient
539
all_revs = self.target.all_revision_ids()
540
parent_map = self.target.get_parent_map(all_revs)
542
for values in viewvalues(parent_map):
543
all_parents.update(values)
544
return set(all_revs) - all_parents
546
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
547
"""See `InterGitNonGitRepository`."""
549
store = get_object_store(self.target, mapping)
550
with store.lock_write():
551
heads = self.get_target_heads()
552
graph_walker = ObjectStoreGraphWalker(
553
[store._lookup_revision_sha1(head) for head in heads],
554
lambda sha: store[sha].parents)
555
wants_recorder = DetermineWantsRecorder(determine_wants)
557
pb = ui.ui_factory.nested_progress_bar()
559
objects_iter = self.source.fetch_objects(
560
wants_recorder, graph_walker, store.get_raw)
561
trace.mutter("Importing %d new revisions",
562
len(wants_recorder.wants))
563
(pack_hint, last_rev) = import_git_objects(self.target,
564
mapping, objects_iter, store, wants_recorder.wants, pb,
566
return (pack_hint, last_rev, wants_recorder.remote_refs)
571
def is_compatible(source, target):
572
"""Be compatible with GitRepository."""
573
if not isinstance(source, RemoteGitRepository):
575
if not target.supports_rich_root():
577
if isinstance(target, GitRepository):
579
if not getattr(target._format, "supports_full_versioned_files", True):
584
class InterLocalGitNonGitRepository(InterGitNonGitRepository):
585
"""InterRepository that copies revisions from a local Git into a non-Git
588
def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
589
"""See `InterGitNonGitRepository`."""
591
remote_refs = self.source.controldir.get_refs_container().as_dict()
592
wants = determine_wants(remote_refs)
594
pb = ui.ui_factory.nested_progress_bar()
595
target_git_object_retriever = get_object_store(self.target, mapping)
597
target_git_object_retriever.lock_write()
599
(pack_hint, last_rev) = import_git_objects(self.target,
600
mapping, self.source._git.object_store,
601
target_git_object_retriever, wants, pb, limit)
602
return (pack_hint, last_rev, remote_refs)
604
target_git_object_retriever.unlock()
609
def is_compatible(source, target):
610
"""Be compatible with GitRepository."""
611
if not isinstance(source, LocalGitRepository):
613
if not target.supports_rich_root():
615
if isinstance(target, GitRepository):
617
if not getattr(target._format, "supports_full_versioned_files", True):
622
class InterGitGitRepository(InterFromGitRepository):
623
"""InterRepository that copies between Git repositories."""
625
def fetch_refs(self, update_refs, lossy, overwrite=False):
627
raise LossyPushToSameVCS(self.source, self.target)
628
old_refs = self.target.controldir.get_refs_container()
630
def determine_wants(heads):
631
old_refs = dict([(k, (v, None)) for (k, v) in viewitems(heads.as_dict())])
632
new_refs = update_refs(old_refs)
633
ref_changes.update(new_refs)
634
return [sha1 for (sha1, bzr_revid) in viewvalues(new_refs)]
635
self.fetch_objects(determine_wants, lossy=lossy)
636
for k, (git_sha, bzr_revid) in viewitems(ref_changes):
637
self.target._git.refs[k] = git_sha
638
new_refs = self.target.controldir.get_refs_container()
639
return None, old_refs, new_refs
641
def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
642
raise NotImplementedError(self.fetch_objects)
644
def _target_has_shas(self, shas):
645
return set([sha for sha in shas if sha in self.target._git.object_store])
647
def fetch(self, revision_id=None, find_ghosts=False,
648
mapping=None, fetch_spec=None, branches=None, limit=None, include_tags=False):
650
mapping = self.source.get_mapping()
651
if revision_id is not None:
653
elif fetch_spec is not None:
654
recipe = fetch_spec.get_recipe()
655
if recipe[0] in ("search", "proxy-search"):
658
raise AssertionError(
659
"Unsupported search result type %s" % recipe[0])
661
if branches is not None:
662
def determine_wants(refs):
664
for name, value in viewitems(refs):
665
if value == ZERO_SHA:
668
if name in branches or (include_tags and is_tag(name)):
671
elif fetch_spec is None and revision_id is None:
672
determine_wants = self.determine_wants_all
674
determine_wants = self.get_determine_wants_revids(args, include_tags=include_tags)
675
wants_recorder = DetermineWantsRecorder(determine_wants)
676
self.fetch_objects(wants_recorder, mapping, limit=limit)
677
return wants_recorder.remote_refs
679
def get_determine_wants_revids(self, revids, include_tags=False):
681
for revid in set(revids):
682
if revid == NULL_REVISION:
684
git_sha, mapping = self.source.lookup_bzr_revision_id(revid)
686
return self.get_determine_wants_heads(wants, include_tags=include_tags)
688
def determine_wants_all(self, refs):
690
v for k, v in refs.items()
691
if not v == ZERO_SHA and not k.endswith(ANNOTATED_TAG_SUFFIX)])
692
return list(potential - self._target_has_shas(potential))
695
class InterLocalGitLocalGitRepository(InterGitGitRepository):
697
def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
699
raise LossyPushToSameVCS(self.source, self.target)
700
if limit is not None:
701
raise FetchLimitUnsupported(self)
702
from .remote import DefaultProgressReporter
703
pb = ui.ui_factory.nested_progress_bar()
704
progress = DefaultProgressReporter(pb).progress
706
refs = self.source._git.fetch(
707
self.target._git, determine_wants,
711
return (None, None, refs)
714
def is_compatible(source, target):
715
"""Be compatible with GitRepository."""
716
return (isinstance(source, LocalGitRepository) and
717
isinstance(target, LocalGitRepository))
720
class InterRemoteGitLocalGitRepository(InterGitGitRepository):
722
def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):
724
raise LossyPushToSameVCS(self.source, self.target)
725
if limit is not None:
726
raise FetchLimitUnsupported(self)
727
graphwalker = self.target._git.get_graph_walker()
728
if CAPABILITY_THIN_PACK in self.source.controldir._client._fetch_capabilities:
729
# TODO(jelmer): Avoid reading entire file into memory and
730
# only processing it after the whole file has been fetched.
736
self.target._git.object_store.move_in_thin_pack(f)
741
f, commit, abort = self.target._git.object_store.add_pack()
743
refs = self.source.controldir.fetch_pack(
744
determine_wants, graphwalker, f.write)
746
return (None, None, refs)
747
except BaseException:
752
def is_compatible(source, target):
753
"""Be compatible with GitRepository."""
754
return (isinstance(source, RemoteGitRepository) and
755
isinstance(target, LocalGitRepository))
added
removed
breezy/git/interrepo.py
