/brz/remove-bazaar

# Copyright (C) 2009-2018 Jelmer Vernooij <jelmer@jelmer.uk>

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

"""InterRepository operations."""

from __future__ import absolute_import

from io import BytesIO

from dulwich.errors import (

    NotCommitError,

    )

from dulwich.object_store import (

    ObjectStoreGraphWalker,

    )

from dulwich.protocol import (

    CAPABILITY_THIN_PACK,

    ZERO_SHA,

    )

from dulwich.walk import Walker

from ...errors import (

    FetchLimitUnsupported,

    InvalidRevisionId,

    LossyPushToSameVCS,

    NoRoundtrippingSupport,

    NoSuchRevision,

    )

from ...repository import (

    InterRepository,

    )

from ...revision import (

    NULL_REVISION,

    )

from ... import (

    trace,

    ui,

    )

from .errors import (

    NoPushSupport,

    )

from .fetch import (

    import_git_objects,

    report_git_progress,

    DetermineWantsRecorder,

    )

from .mapping import (

    needs_roundtripping,

    )

from .object_store import (

    get_object_store,

    _tree_to_objects,

    )

from .push import (

    MissingObjectsIterator,

    )

from .refs import (

    is_tag,

    )

from .repository import (

    GitRepository,

    LocalGitRepository,

    GitRepositoryFormat,

    )

from .remote import (

    RemoteGitRepository,

    )

from .unpeel_map import (

    UnpeelMap,

    )

class InterToGitRepository(InterRepository):

    """InterRepository that copies into a Git repository."""

    _matching_repo_format = GitRepositoryFormat()

    def __init__(self, source, target):

        super(InterToGitRepository, self).__init__(source, target)

        self.mapping = self.target.get_mapping()

        self.source_store = get_object_store(self.source, self.mapping)

    @staticmethod

    def _get_repo_format_to_test():

        return None

    def copy_content(self, revision_id=None, pb=None):

        """See InterRepository.copy_content."""

        self.fetch(revision_id, pb, find_ghosts=False)

    def fetch_refs(self, update_refs, lossy):

        """Fetch possibly roundtripped revisions into the target repository

        and update refs.

        :param update_refs: Generate refs to fetch. Receives dictionary

            with old refs (git shas), returns dictionary of new names to

            git shas.

        :param lossy: Whether to roundtrip

        :return: old refs, new refs

        """

        raise NotImplementedError(self.fetch_refs)

    def search_missing_revision_ids(self,

            find_ghosts=True, revision_ids=None, if_present_ids=None,

            limit=None):

        if limit is not None:

            raise FetchLimitUnsupported(self)

        git_shas = []

        todo = []

        if revision_ids:

            todo.extend(revision_ids)

        if if_present_ids:

            todo.extend(revision_ids)

        with self.source_store.lock_read():

            for revid in revision_ids:

                if revid == NULL_REVISION:

                    continue

                git_sha = self.source_store._lookup_revision_sha1(revid)

                git_shas.append(git_sha)

            walker = Walker(self.source_store,

                include=git_shas, exclude=[

                    sha for sha in self.target.controldir.get_refs_container().as_dict().values()

                    if sha != ZERO_SHA])

            missing_revids = set()

            for entry in walker:

                for (kind, type_data) in self.source_store.lookup_git_sha(entry.commit.id):

                    if kind == "commit":

                        missing_revids.add(type_data[0])

        return self.source.revision_ids_to_search_result(missing_revids)

    def _warn_slow(self):

        trace.warning(

            'Pushing from a Bazaar to a Git repository. '

            'For better performance, push into a Bazaar repository.')

class InterToLocalGitRepository(InterToGitRepository):

    """InterBranch implementation between a Bazaar and a Git repository."""

    def __init__(self, source, target):

        super(InterToLocalGitRepository, self).__init__(source, target)

        self.target_store = self.target.controldir._git.object_store

        self.target_refs = self.target.controldir._git.refs

    def _commit_needs_fetching(self, sha_id):

        try:

            return (sha_id not in self.target_store)

        except NoSuchRevision:

            # Ghost, can't push

            return False

    def _revision_needs_fetching(self, sha_id, revid):

        if revid == NULL_REVISION:

            return False

        if sha_id is None:

            try:

                sha_id = self.source_store._lookup_revision_sha1(revid)

            except KeyError:

                return False

        return self._commit_needs_fetching(sha_id)

    def missing_revisions(self, stop_revisions):

        """Find the revisions that are missing from the target repository.

        :param stop_revisions: Revisions to check for (tuples with

            Git SHA1, bzr revid)

        :return: sequence of missing revisions, in topological order

        :raise: NoSuchRevision if the stop_revisions are not present in

            the source

        """

        revid_sha_map = {}

        stop_revids = []

        for (sha1, revid) in stop_revisions:

            if sha1 is not None and revid is not None:

                revid_sha_map[revid] = sha1

                stop_revids.append(revid)

            elif sha1 is not None:

                if self._commit_needs_fetching(sha1):

                    for (kind, (revid, tree_sha, verifiers)) in self.source_store.lookup_git_sha(sha1):

                        revid_sha_map[revid] = sha1

                        stop_revids.append(revid)

            else:

                if revid is None:

                    raise AssertionError

                stop_revids.append(revid)

        missing = set()

        graph = self.source.get_graph()

        pb = ui.ui_factory.nested_progress_bar()

        try:

            while stop_revids:

                new_stop_revids = []

                for revid in stop_revids:

                    sha1 = revid_sha_map.get(revid)

                    if (not revid in missing and

                        self._revision_needs_fetching(sha1, revid)):

                        missing.add(revid)

                        new_stop_revids.append(revid)

                stop_revids = set()

                parent_map = graph.get_parent_map(new_stop_revids)

                for parent_revids in parent_map.itervalues():

                    stop_revids.update(parent_revids)

                pb.update("determining revisions to fetch", len(missing))

        finally:

            pb.finished()

        return graph.iter_topo_order(missing)

    def _get_target_bzr_refs(self):

        """Return a dictionary with references.

        :return: Dictionary with reference names as keys and tuples

            with Git SHA, Bazaar revid as values.

        """

        bzr_refs = {}

        refs = {}

        for k in self.target._git.refs.allkeys():

            try:

                v = self.target._git.refs[k]

            except KeyError:

                # broken symref?

                continue

            try:

                for (kind, type_data) in self.source_store.lookup_git_sha(v):

                    if kind == "commit" and self.source.has_revision(type_data[0]):

                        revid = type_data[0]

                        break

                else:

                    revid = None

            except KeyError:

                revid = None

            bzr_refs[k] = (v, revid)

        return bzr_refs

    def fetch_refs(self, update_refs, lossy):

        self._warn_slow()

        with self.source_store.lock_read():

            old_refs = self._get_target_bzr_refs()

            new_refs = update_refs(old_refs)

            revidmap = self.fetch_objects(

                [(git_sha, bzr_revid) for (git_sha, bzr_revid) in new_refs.values() if git_sha is None or not git_sha.startswith('ref:')], lossy=lossy)

            for name, (gitid, revid) in new_refs.iteritems():

                if gitid is None:

                    try:

                        gitid = revidmap[revid][0]

                    except KeyError:

                        gitid = self.source_store._lookup_revision_sha1(revid)

                if len(gitid) != 40 and not gitid.startswith('ref: '):

                    raise AssertionError("invalid ref contents: %r" % gitid)

                self.target_refs[name] = gitid

        return revidmap, old_refs, new_refs

    def fetch_objects(self, revs, lossy, limit=None):

        if not lossy and not self.mapping.roundtripping:

            for git_sha, bzr_revid in revs:

                if bzr_revid is not None and needs_roundtripping(self.source, bzr_revid):

                    raise NoPushSupport(self.source, self.target, self.mapping,

                                        bzr_revid)

        with self.source_store.lock_read():

            todo = list(self.missing_revisions(revs))[:limit]

            revidmap = {}

            pb = ui.ui_factory.nested_progress_bar()

            try:

                object_generator = MissingObjectsIterator(

                    self.source_store, self.source, pb)

                for (old_revid, git_sha) in object_generator.import_revisions(

                    todo, lossy=lossy):

                    if lossy:

                        new_revid = self.mapping.revision_id_foreign_to_bzr(git_sha)

                    else:

                        new_revid = old_revid

                        try:

                            self.mapping.revision_id_bzr_to_foreign(old_revid)

                        except InvalidRevisionId:

                            refname = self.mapping.revid_as_refname(old_revid)

                            self.target_refs[refname] = git_sha

                    revidmap[old_revid] = (git_sha, new_revid)

                self.target_store.add_objects(object_generator)

                return revidmap

            finally:

                pb.finished()

    def fetch(self, revision_id=None, pb=None, find_ghosts=False,

            fetch_spec=None, mapped_refs=None):

        if mapped_refs is not None:

            stop_revisions = mapped_refs

        elif revision_id is not None:

            stop_revisions = [(None, revision_id)]

        elif fetch_spec is not None:

            recipe = fetch_spec.get_recipe()

            if recipe[0] in ("search", "proxy-search"):

                stop_revisions = [(None, revid) for revid in recipe[1]]

            else:

                raise AssertionError("Unsupported search result type %s" % recipe[0])

        else:

            stop_revisions = [(None, revid) for revid in self.source.all_revision_ids()]

        self._warn_slow()

        try:

            self.fetch_objects(stop_revisions, lossy=False)

        except NoPushSupport:

            raise NoRoundtrippingSupport(self.source, self.target)

    @staticmethod

    def is_compatible(source, target):

        """Be compatible with GitRepository."""

        return (not isinstance(source, GitRepository) and

                isinstance(target, LocalGitRepository))

class InterToRemoteGitRepository(InterToGitRepository):

    def fetch_refs(self, update_refs, lossy):

        """Import the gist of the ancestry of a particular revision."""

        if not lossy and not self.mapping.roundtripping:

            raise NoPushSupport(self.source, self.target, self.mapping)

        unpeel_map = UnpeelMap.from_repository(self.source)

        revidmap = {}

        def determine_wants(old_refs):

            ret = {}

            self.old_refs = dict([(k, (v, None)) for (k, v) in old_refs.iteritems()])

            self.new_refs = update_refs(self.old_refs)

            for name, (gitid, revid) in self.new_refs.iteritems():

                if gitid is None:

                    git_sha = self.source_store._lookup_revision_sha1(revid)

                    ret[name] = unpeel_map.re_unpeel_tag(git_sha, old_refs.get(name))

                else:

                    ret[name] = gitid

            return ret

        self._warn_slow()

        with self.source_store.lock_read():

            new_refs = self.target.send_pack(determine_wants,

                    self.source_store.generate_lossy_pack_data)

        # FIXME: revidmap?

        return revidmap, self.old_refs, self.new_refs

    @staticmethod

    def is_compatible(source, target):

        """Be compatible with GitRepository."""

        return (not isinstance(source, GitRepository) and

                isinstance(target, RemoteGitRepository))

class InterFromGitRepository(InterRepository):

    _matching_repo_format = GitRepositoryFormat()

    def _target_has_shas(self, shas):

        raise NotImplementedError(self._target_has_shas)

    def get_determine_wants_heads(self, wants, include_tags=False):

        wants = set(wants)

        def determine_wants(refs):

            potential = set(wants)

            if include_tags:

                for k, unpeeled in refs.iteritems():

                    if k.endswith("^{}"):

                        continue

                    if not is_tag(k):

                        continue

                    if unpeeled == ZERO_SHA:

                        continue

                    potential.add(unpeeled)

            return list(potential - self._target_has_shas(potential))

        return determine_wants

    def determine_wants_all(self, refs):

        raise NotImplementedError(self.determine_wants_all)

    @staticmethod

    def _get_repo_format_to_test():

        return None

    def copy_content(self, revision_id=None):

        """See InterRepository.copy_content."""

        self.fetch(revision_id, find_ghosts=False)

    def search_missing_revision_ids(self,

            find_ghosts=True, revision_ids=None, if_present_ids=None,

            limit=None):

        if limit is not None:

            raise FetchLimitUnsupported(self)

        git_shas = []

        todo = []

        if revision_ids:

            todo.extend(revision_ids)

        if if_present_ids:

            todo.extend(revision_ids)

        with self.lock_read():

            for revid in revision_ids:

                if revid == NULL_REVISION:

                    continue

                git_sha, mapping = self.source.lookup_bzr_revision_id(revid)

                git_shas.append(git_sha)

            walker = Walker(self.source._git.object_store,

                include=git_shas, exclude=[

                    sha for sha in self.target.controldir.get_refs_container().as_dict().values()

                    if sha != ZERO_SHA])

            missing_revids = set()

            for entry in walker:

                missing_revids.add(self.source.lookup_foreign_revision_id(entry.commit.id))

            return self.source.revision_ids_to_search_result(missing_revids)

class InterGitNonGitRepository(InterFromGitRepository):

    """Base InterRepository that copies revisions from a Git into a non-Git

    repository."""

    def _target_has_shas(self, shas):

        revids = {}

        for sha in shas:

            try:

                revid = self.source.lookup_foreign_revision_id(sha)

            except NotCommitError:

                # Commit is definitely not present

                continue

            else:

                revids[revid] = sha

        return set([revids[r] for r in self.target.has_revisions(revids)])

    def determine_wants_all(self, refs):

        potential = set()

        for k, v in refs.iteritems():

            # For non-git target repositories, only worry about peeled

            if v == ZERO_SHA:

                continue

            potential.add(self.source.controldir.get_peeled(k) or v)

        return list(potential - self._target_has_shas(potential))

    def get_determine_wants_heads(self, wants, include_tags=False):

        wants = set(wants)

        def determine_wants(refs):

            potential = set(wants)

            if include_tags:

                for k, unpeeled in refs.iteritems():

                    if not is_tag(k):

                        continue

                    if unpeeled == ZERO_SHA:

                        continue

                    potential.add(self.source.controldir.get_peeled(k) or unpeeled)

            return list(potential - self._target_has_shas(potential))

        return determine_wants

    def _warn_slow(self):

        trace.warning(

            'Fetching from Git to Bazaar repository. '

            'For better performance, fetch into a Git repository.')

    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):

        """Fetch objects from a remote server.

        :param determine_wants: determine_wants callback

        :param mapping: BzrGitMapping to use

        :param limit: Maximum number of commits to import.

        :return: Tuple with pack hint, last imported revision id and remote refs

        """

        raise NotImplementedError(self.fetch_objects)

    def get_determine_wants_revids(self, revids, include_tags=False):

        wants = set()

        for revid in set(revids):

            if self.target.has_revision(revid):

                continue

            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)

            wants.add(git_sha)

        return self.get_determine_wants_heads(wants, include_tags=include_tags)

    def fetch(self, revision_id=None, find_ghosts=False,

              mapping=None, fetch_spec=None, include_tags=False):

        if mapping is None:

            mapping = self.source.get_mapping()

        if revision_id is not None:

            interesting_heads = [revision_id]

        elif fetch_spec is not None:

            recipe = fetch_spec.get_recipe()

            if recipe[0] in ("search", "proxy-search"):

                interesting_heads = recipe[1]

            else:

                raise AssertionError("Unsupported search result type %s" %

                        recipe[0])

        else:

            interesting_heads = None

        if interesting_heads is not None:

            determine_wants = self.get_determine_wants_revids(

                interesting_heads, include_tags=include_tags)

        else:

            determine_wants = self.determine_wants_all

        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,

            mapping)

        if pack_hint is not None and self.target._format.pack_compresses:

            self.target.pack(hint=pack_hint)

        return remote_refs

class InterRemoteGitNonGitRepository(InterGitNonGitRepository):

    """InterRepository that copies revisions from a remote Git into a non-Git

    repository."""

    def get_target_heads(self):

        # FIXME: This should be more efficient

        all_revs = self.target.all_revision_ids()

        parent_map = self.target.get_parent_map(all_revs)

        all_parents = set()

        map(all_parents.update, parent_map.itervalues())

        return set(all_revs) - all_parents

    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):

        """See `InterGitNonGitRepository`."""

        self._warn_slow()

        store = get_object_store(self.target, mapping)

        with store.lock_write():

            heads = self.get_target_heads()

            graph_walker = ObjectStoreGraphWalker(

                [store._lookup_revision_sha1(head) for head in heads],

                lambda sha: store[sha].parents)

            wants_recorder = DetermineWantsRecorder(determine_wants)

            pb = ui.ui_factory.nested_progress_bar()

            try:

                objects_iter = self.source.fetch_objects(

                    wants_recorder, graph_walker, store.get_raw,

                    progress=lambda text: report_git_progress(pb, text),)

                trace.mutter("Importing %d new revisions",

                             len(wants_recorder.wants))

                (pack_hint, last_rev) = import_git_objects(self.target,

                    mapping, objects_iter, store, wants_recorder.wants, pb,

                    limit)

                return (pack_hint, last_rev, wants_recorder.remote_refs)

            finally:

                pb.finished()

    @staticmethod

    def is_compatible(source, target):

        """Be compatible with GitRepository."""

        if not isinstance(source, RemoteGitRepository):

            return False

        if not target.supports_rich_root():

            return False

        if isinstance(target, GitRepository):

            return False

        if not getattr(target._format, "supports_full_versioned_files", True):

            return False

        return True

class InterLocalGitNonGitRepository(InterGitNonGitRepository):

    """InterRepository that copies revisions from a local Git into a non-Git

    repository."""

    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):

        """See `InterGitNonGitRepository`."""

        self._warn_slow()

        remote_refs = self.source.controldir.get_refs_container().as_dict()

        wants = determine_wants(remote_refs)

        create_pb = None

        pb = ui.ui_factory.nested_progress_bar()

        target_git_object_retriever = get_object_store(self.target, mapping)

        try:

            target_git_object_retriever.lock_write()

            try:

                (pack_hint, last_rev) = import_git_objects(self.target,

                    mapping, self.source._git.object_store,

                    target_git_object_retriever, wants, pb, limit)

                return (pack_hint, last_rev, remote_refs)

            finally:

                target_git_object_retriever.unlock()

        finally:

            pb.finished()

    @staticmethod

    def is_compatible(source, target):

        """Be compatible with GitRepository."""

        if not isinstance(source, LocalGitRepository):

            return False

        if not target.supports_rich_root():

            return False

        if isinstance(target, GitRepository):

            return False

        if not getattr(target._format, "supports_full_versioned_files", True):

            return False

        return True

class InterGitGitRepository(InterFromGitRepository):

    """InterRepository that copies between Git repositories."""

    def fetch_refs(self, update_refs, lossy):

        if lossy:

            raise LossyPushToSameVCS(self.source, self.target)

        old_refs = self.target.controldir.get_refs_container()

        ref_changes = {}

        def determine_wants(heads):

            old_refs = dict([(k, (v, None)) for (k, v) in heads.as_dict().iteritems()])

            new_refs = update_refs(old_refs)

            ref_changes.update(new_refs)

            return [sha1 for (sha1, bzr_revid) in new_refs.itervalues()]

        self.fetch_objects(determine_wants, lossy=lossy)

        for k, (git_sha, bzr_revid) in ref_changes.iteritems():

            self.target._git.refs[k] = git_sha

        new_refs = self.target.controldir.get_refs_container()

        return None, old_refs, new_refs

    def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):

        raise NotImplementedError(self.fetch_objects)

    def _target_has_shas(self, shas):

        return set([sha for sha in shas if sha in self.target._git.object_store])

    def fetch(self, revision_id=None, find_ghosts=False,

              mapping=None, fetch_spec=None, branches=None, limit=None, include_tags=False):

        if mapping is None:

            mapping = self.source.get_mapping()

        if revision_id is not None:

            args = [revision_id]

        elif fetch_spec is not None:

            recipe = fetch_spec.get_recipe()

            if recipe[0] in ("search", "proxy-search"):

                heads = recipe[1]

            else:

                raise AssertionError(

                    "Unsupported search result type %s" % recipe[0])

            args = heads

        if branches is not None:

            def determine_wants(refs):

                ret = []

                for name, value in refs.iteritems():

                    if value == ZERO_SHA:

                        continue

                    if name in branches or (include_tags and is_tag(name)):

                        ret.append(value)

                return ret

        elif fetch_spec is None and revision_id is None:

            determine_wants = self.determine_wants_all

        else:

            determine_wants = self.get_determine_wants_revids(args, include_tags=include_tags)

        wants_recorder = DetermineWantsRecorder(determine_wants)

        self.fetch_objects(wants_recorder, mapping, limit=limit)

        return wants_recorder.remote_refs

    def get_determine_wants_revids(self, revids, include_tags=False):

        wants = set()

        for revid in set(revids):

            if revid == NULL_REVISION:

                continue

            git_sha, mapping = self.source.lookup_bzr_revision_id(revid)

            wants.add(git_sha)

        return self.get_determine_wants_heads(wants, include_tags=include_tags)

    def determine_wants_all(self, refs):

        potential = set([v for v in refs.values() if not v == ZERO_SHA])

        return list(potential - self._target_has_shas(potential))

class InterLocalGitLocalGitRepository(InterGitGitRepository):

    def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):

        if lossy:

            raise LossyPushToSameVCS(self.source, self.target)

        if limit is not None:

            raise FetchLimitUnsupported(self)

        pb = ui.ui_factory.nested_progress_bar()

        try:

            refs = self.source._git.fetch(self.target._git, determine_wants,

                lambda text: report_git_progress(pb, text))

        finally:

            pb.finished()

        return (None, None, refs)

    @staticmethod

    def is_compatible(source, target):

        """Be compatible with GitRepository."""

        return (isinstance(source, LocalGitRepository) and

                isinstance(target, LocalGitRepository))

class InterRemoteGitLocalGitRepository(InterGitGitRepository):

    def fetch_objects(self, determine_wants, mapping=None, limit=None, lossy=False):

        if lossy:

            raise LossyPushToSameVCS(self.source, self.target)

        if limit is not None:

            raise FetchLimitUnsupported(self)

        graphwalker = self.target._git.get_graph_walker()

        pb = ui.ui_factory.nested_progress_bar()

        try:

            if CAPABILITY_THIN_PACK in self.source.controldir._client._fetch_capabilities:

                # TODO(jelmer): Avoid reading entire file into memory and

                # only processing it after the whole file has been fetched.

                f = BytesIO()

                def commit():

                    if f.tell():

                        f.seek(0)

                        self.target._git.object_store.move_in_thin_pack(f)

                def abort():

                    pass

            else: