/brz/remove-bazaar

# Copyright (C) 2009-2018 Jelmer Vernooij <jelmer@jelmer.uk>

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

"""Map from Git sha's to Bazaar objects."""

from __future__ import absolute_import

from dulwich.objects import (

    sha_to_hex,

    hex_to_sha,

    )

import os

import threading

from dulwich.objects import (

    ShaFile,

    )

from ... import (

    errors as bzr_errors,

    osutils,

    registry,

    trace,

    )

from ...bzr import (

    btree_index as _mod_btree_index,

    index as _mod_index,

    versionedfile,

    )

from ...sixish import (

    viewitems,

    viewkeys,

    viewvalues,

    )

from ...transport import (

    get_transport,

    )

def get_cache_dir():

    try:

        from xdg.BaseDirectory import xdg_cache_home

    except ImportError:

        from ...config import config_dir

        ret = os.path.join(config_dir(), "git")

    else:

        ret = os.path.join(xdg_cache_home, "breezy", "git")

    if not os.path.isdir(ret):

        os.makedirs(ret)

    return ret

def get_remote_cache_transport(repository):

    """Retrieve the transport to use when accessing (unwritable) remote 

    repositories.

    """

    uuid = getattr(repository, "uuid", None)

    if uuid is None:

        path = get_cache_dir()

    else:

        path = os.path.join(get_cache_dir(), uuid)

        if not os.path.isdir(path):

            os.mkdir(path)

    return get_transport(path)

def check_pysqlite_version(sqlite3):

    """Check that sqlite library is compatible.

    """

    if (sqlite3.sqlite_version_info[0] < 3 or

            (sqlite3.sqlite_version_info[0] == 3 and

             sqlite3.sqlite_version_info[1] < 3)):

        trace.warning('Needs at least sqlite 3.3.x')

        raise bzr_errors.BzrError("incompatible sqlite library")

try:

    try:

        import sqlite3

        check_pysqlite_version(sqlite3)

    except (ImportError, bzr_errors.BzrError) as e:

        from pysqlite2 import dbapi2 as sqlite3

        check_pysqlite_version(sqlite3)

except:

    trace.warning('Needs at least Python2.5 or Python2.4 with the pysqlite2 '

            'module')

    raise bzr_errors.BzrError("missing sqlite library")

_mapdbs = threading.local()

def mapdbs():

    """Get a cache for this thread's db connections."""

    try:

        return _mapdbs.cache

    except AttributeError:

        _mapdbs.cache = {}

        return _mapdbs.cache

class GitShaMap(object):

    """Git<->Bzr revision id mapping database."""

    def lookup_git_sha(self, sha):

        """Lookup a Git sha in the database.

        :param sha: Git object sha

        :return: list with (type, type_data) tuples with type_data:

            commit: revid, tree_sha, verifiers

            blob: fileid, revid

            tree: fileid, revid

        """

        raise NotImplementedError(self.lookup_git_sha)

    def lookup_blob_id(self, file_id, revision):

        """Retrieve a Git blob SHA by file id.

        :param file_id: File id of the file/symlink

        :param revision: revision in which the file was last changed.

        """

        raise NotImplementedError(self.lookup_blob_id)

    def lookup_tree_id(self, file_id, revision):

        """Retrieve a Git tree SHA by file id.

        """

        raise NotImplementedError(self.lookup_tree_id)

    def lookup_commit(self, revid):

        """Retrieve a Git commit SHA by Bazaar revision id.

        """

        raise NotImplementedError(self.lookup_commit)

    def revids(self):

        """List the revision ids known."""

        raise NotImplementedError(self.revids)

    def missing_revisions(self, revids):

        """Return set of all the revisions that are not present."""

        present_revids = set(self.revids())

        if not isinstance(revids, set):

            revids = set(revids)

        return revids - present_revids

    def sha1s(self):

        """List the SHA1s."""

        raise NotImplementedError(self.sha1s)

    def start_write_group(self):

        """Start writing changes."""

    def commit_write_group(self):

        """Commit any pending changes."""

    def abort_write_group(self):

        """Abort any pending changes."""

class ContentCache(object):

    """Object that can cache Git objects."""

    def add(self, object):

        """Add an object."""

        raise NotImplementedError(self.add)

    def add_multi(self, objects):

        """Add multiple objects."""

        for obj in objects:

            self.add(obj)

    def __getitem__(self, sha):

        """Retrieve an item, by SHA."""

        raise NotImplementedError(self.__getitem__)

class BzrGitCacheFormat(object):

    """Bazaar-Git Cache Format."""

    def get_format_string(self):

        """Return a single-line unique format string for this cache format."""

        raise NotImplementedError(self.get_format_string)

    def open(self, transport):

        """Open this format on a transport."""

        raise NotImplementedError(self.open)

    def initialize(self, transport):

        """Create a new instance of this cache format at transport."""

        transport.put_bytes('format', self.get_format_string())

    @classmethod

    def from_transport(self, transport):

        """Open a cache file present on a transport, or initialize one.

        :param transport: Transport to use

        :return: A BzrGitCache instance

        """

        try:

            format_name = transport.get_bytes('format')

            format = formats.get(format_name)

        except bzr_errors.NoSuchFile:

            format = formats.get('default')

            format.initialize(transport)

        return format.open(transport)

    @classmethod

    def from_repository(cls, repository):

        """Open a cache file for a repository.

        This will use the repository's transport to store the cache file, or

        use the users global cache directory if the repository has no 

        transport associated with it.

        :param repository: Repository to open the cache for

        :return: A `BzrGitCache`

        """

        from ...transport.local import LocalTransport

        repo_transport = getattr(repository, "_transport", None)

        if (repo_transport is not None and

            isinstance(repo_transport, LocalTransport)):

            # Even if we don't write to this repo, we should be able

            # to update its cache.

            try:

                repo_transport = remove_readonly_transport_decorator(repo_transport)

            except bzr_errors.ReadOnlyError:

                transport = None

            else:

                try:

                    repo_transport.mkdir('git')

                except bzr_errors.FileExists:

                    pass

                transport = repo_transport.clone('git')

        else:

            transport = None

        if transport is None:

            transport = get_remote_cache_transport(repository)

        return cls.from_transport(transport)

class CacheUpdater(object):

    """Base class for objects that can update a bzr-git cache."""

    def add_object(self, obj, bzr_key_data, path):

        """Add an object.

        :param obj: Object type ("commit", "blob" or "tree")

        :param bzr_key_data: bzr key store data or testament_sha in case

            of commit

        :param path: Path of the object (optional)

        """

        raise NotImplementedError(self.add_object)

    def finish(self):

        raise NotImplementedError(self.finish)

class BzrGitCache(object):

    """Caching backend."""

    def __init__(self, idmap, cache_updater_klass):

        self.idmap = idmap

        self._cache_updater_klass = cache_updater_klass

    def get_updater(self, rev):

        """Update an object that implements the CacheUpdater interface for 

        updating this cache.

        """

        return self._cache_updater_klass(self, rev)

DictBzrGitCache = lambda: BzrGitCache(DictGitShaMap(), DictCacheUpdater)

class DictCacheUpdater(CacheUpdater):

    """Cache updater for dict-based caches."""

    def __init__(self, cache, rev):

        self.cache = cache

        self.revid = rev.revision_id

        self.parent_revids = rev.parent_ids

        self._commit = None

        self._entries = []

    def add_object(self, obj, bzr_key_data, path):

        if isinstance(obj, tuple):

            (type_name, hexsha) = obj

        else:

            type_name = obj.type_name.decode('ascii')

            hexsha = obj.id

        if type_name == "commit":

            self._commit = obj

            if type(bzr_key_data) is not dict:

                raise TypeError(bzr_key_data)

            key = self.revid

            type_data = (self.revid, self._commit.tree, bzr_key_data)

            self.cache.idmap._by_revid[self.revid] = hexsha

        elif type_name in ("blob", "tree"):

            if bzr_key_data is not None:

                key = type_data = bzr_key_data

                self.cache.idmap._by_fileid.setdefault(type_data[1], {})[type_data[0]] = hexsha

        else:

            raise AssertionError

        entry = (type_name, type_data)

        self.cache.idmap._by_sha.setdefault(hexsha, {})[key] = entry

    def finish(self):

        if self._commit is None:

            raise AssertionError("No commit object added")

        return self._commit

class DictGitShaMap(GitShaMap):

    """Git SHA map that uses a dictionary."""

    def __init__(self):

        self._by_sha = {}

        self._by_fileid = {}

        self._by_revid = {}

    def lookup_blob_id(self, fileid, revision):

        return self._by_fileid[revision][fileid]

    def lookup_git_sha(self, sha):

        for entry in viewvalues(self._by_sha[sha]):

            yield entry

    def lookup_tree_id(self, fileid, revision):

        return self._by_fileid[revision][fileid]

    def lookup_commit(self, revid):

        return self._by_revid[revid]

    def revids(self):

        for key, entries in viewitems(self._by_sha):

            for (type, type_data) in entries.values():

                if type == "commit":

                    yield type_data[0]

    def sha1s(self):

        return viewkeys(self._by_sha)

class SqliteCacheUpdater(CacheUpdater):

    def __init__(self, cache, rev):

        self.cache = cache

        self.db = self.cache.idmap.db

        self.revid = rev.revision_id

        self._commit = None

        self._trees = []

        self._blobs = []

    def add_object(self, obj, bzr_key_data, path):

        if isinstance(obj, tuple):

            (type_name, hexsha) = obj

        else:

            type_name = obj.type_name.decode('ascii')

            hexsha = obj.id

        if type_name == "commit":

            self._commit = obj

            if type(bzr_key_data) is not dict:

                raise TypeError(bzr_key_data)

            self._testament3_sha1 = bzr_key_data.get("testament3-sha1")

        elif type_name == "tree":

            if bzr_key_data is not None:

                self._trees.append((hexsha, bzr_key_data[0], bzr_key_data[1]))

        elif type_name == "blob":

            if bzr_key_data is not None:

                self._blobs.append((hexsha, bzr_key_data[0], bzr_key_data[1]))

        else:

            raise AssertionError

    def finish(self):

        if self._commit is None:

            raise AssertionError("No commit object added")

        self.db.executemany(

            "replace into trees (sha1, fileid, revid) values (?, ?, ?)",

            self._trees)

        self.db.executemany(

            "replace into blobs (sha1, fileid, revid) values (?, ?, ?)",

            self._blobs)

        self.db.execute(

            "replace into commits (sha1, revid, tree_sha, testament3_sha1) values (?, ?, ?, ?)",

            (self._commit.id, self.revid, self._commit.tree, self._testament3_sha1))

        return self._commit

SqliteBzrGitCache = lambda p: BzrGitCache(SqliteGitShaMap(p), SqliteCacheUpdater)

class SqliteGitCacheFormat(BzrGitCacheFormat):

    def get_format_string(self):

        return b'bzr-git sha map version 1 using sqlite\n'

    def open(self, transport):

        try:

            basepath = transport.local_abspath(".")

        except bzr_errors.NotLocalUrl:

            basepath = get_cache_dir()

        return SqliteBzrGitCache(os.path.join(basepath, "idmap.db"))

class SqliteGitShaMap(GitShaMap):

    """Bazaar GIT Sha map that uses a sqlite database for storage."""

    def __init__(self, path=None):

        self.path = path

        if path is None:

            self.db = sqlite3.connect(":memory:")

        else:

            if path not in mapdbs():

                mapdbs()[path] = sqlite3.connect(path)

            self.db = mapdbs()[path]

        self.db.text_factory = str

        self.db.executescript("""

        create table if not exists commits(

            sha1 text not null check(length(sha1) == 40),

            revid text not null,

            tree_sha text not null check(length(tree_sha) == 40)

        );

        create index if not exists commit_sha1 on commits(sha1);

        create unique index if not exists commit_revid on commits(revid);

        create table if not exists blobs(

            sha1 text not null check(length(sha1) == 40),

            fileid text not null,

            revid text not null

        );

        create index if not exists blobs_sha1 on blobs(sha1);

        create unique index if not exists blobs_fileid_revid on blobs(fileid, revid);

        create table if not exists trees(

            sha1 text unique not null check(length(sha1) == 40),

            fileid text not null,

            revid text not null

        );

        create unique index if not exists trees_sha1 on trees(sha1);

        create unique index if not exists trees_fileid_revid on trees(fileid, revid);

""")

        try:

            self.db.executescript(

                "ALTER TABLE commits ADD testament3_sha1 TEXT;")

        except sqlite3.OperationalError:

            pass # Column already exists.

    def __repr__(self):

        return "%s(%r)" % (self.__class__.__name__, self.path)

    def lookup_commit(self, revid):

        cursor = self.db.execute("select sha1 from commits where revid = ?", 

            (revid,))

        row = cursor.fetchone()

        if row is not None:

            return row[0]

        raise KeyError

    def commit_write_group(self):

        self.db.commit()

    def lookup_blob_id(self, fileid, revision):

        row = self.db.execute("select sha1 from blobs where fileid = ? and revid = ?", (fileid, revision)).fetchone()

        if row is not None:

            return row[0]

        raise KeyError(fileid)

    def lookup_tree_id(self, fileid, revision):

        row = self.db.execute("select sha1 from trees where fileid = ? and revid = ?", (fileid, revision)).fetchone()

        if row is not None:

            return row[0]

        raise KeyError(fileid)

    def lookup_git_sha(self, sha):

        """Lookup a Git sha in the database.

        :param sha: Git object sha

        :return: (type, type_data) with type_data:

            commit: revid, tree sha, verifiers

            tree: fileid, revid

            blob: fileid, revid

        """

        found = False

        cursor = self.db.execute("select revid, tree_sha, testament3_sha1 from commits where sha1 = ?", (sha,))

        for row in cursor.fetchall():

            found = True

            if row[2] is not None:

                verifiers = {"testament3-sha1": row[2]}

            else:

                verifiers = {}

            yield ("commit", (row[0], row[1], verifiers))

        cursor = self.db.execute("select fileid, revid from blobs where sha1 = ?", (sha,))

        for row in cursor.fetchall():

            found = True

            yield ("blob", row)

        cursor = self.db.execute("select fileid, revid from trees where sha1 = ?", (sha,))

        for row in cursor.fetchall():

            found = True

            yield ("tree", row)

        if not found:

            raise KeyError(sha)

    def revids(self):

        """List the revision ids known."""

        return (row for (row,) in self.db.execute("select revid from commits"))

    def sha1s(self):

        """List the SHA1s."""

        for table in ("blobs", "commits", "trees"):

            for (sha,) in self.db.execute("select sha1 from %s" % table):

                yield sha

class TdbCacheUpdater(CacheUpdater):

    """Cache updater for tdb-based caches."""

    def __init__(self, cache, rev):

        self.cache = cache

        self.db = cache.idmap.db

        self.revid = rev.revision_id

        self.parent_revids = rev.parent_ids

        self._commit = None

        self._entries = []

    def add_object(self, obj, bzr_key_data, path):

        if isinstance(obj, tuple):

            (type_name, hexsha) = obj

            sha = hex_to_sha(hexsha)

        else:

            type_name = obj.type_name.decode('ascii')

            sha = obj.sha().digest()

        if type_name == "commit":

            self.db[b"commit\0" + self.revid] = b"\0".join((sha, obj.tree))

            if type(bzr_key_data) is not dict:

                raise TypeError(bzr_key_data)

            type_data = (self.revid, obj.tree)

            try:

                type_data += (bzr_key_data["testament3-sha1"],)

            except KeyError:

                pass

            self._commit = obj

        elif type_name == "blob":

            if bzr_key_data is None:

                return

            self.db[b"\0".join((b"blob", bzr_key_data[0], bzr_key_data[1]))] = sha

            type_data = bzr_key_data

        elif type_name == "tree":

            if bzr_key_data is None:

                return

            type_data = bzr_key_data

        else:

            raise AssertionError

        entry = b"\0".join((type_name.encode('ascii'), ) + type_data) + b"\n"

        key = b"git\0" + sha

        try:

            oldval = self.db[key]

        except KeyError:

            self.db[key] = entry

        else:

            if not oldval.endswith(b'\n'):

                self.db[key] = b"".join([oldval, b"\n", entry])

            else:

                self.db[key] = b"".join([oldval, entry])

    def finish(self):

        if self._commit is None:

            raise AssertionError("No commit object added")

        return self._commit

TdbBzrGitCache = lambda p: BzrGitCache(TdbGitShaMap(p), TdbCacheUpdater)

class TdbGitCacheFormat(BzrGitCacheFormat):

    """Cache format for tdb-based caches."""

    def get_format_string(self):

        return b'bzr-git sha map version 3 using tdb\n'

    def open(self, transport):

        try:

            basepath = transport.local_abspath(".").encode(osutils._fs_enc)

        except bzr_errors.NotLocalUrl:

            basepath = get_cache_dir()

        if not isinstance(basepath, str):

            raise TypeError(basepath)

        try:

            return TdbBzrGitCache(os.path.join(basepath, "idmap.tdb"))

        except ImportError:

            raise ImportError(

                "Unable to open existing bzr-git cache because 'tdb' is not "

                "installed.")

class TdbGitShaMap(GitShaMap):

    """SHA Map that uses a TDB database.

    Entries:

    "git <sha1>" -> "<type> <type-data1> <type-data2>"

    "commit revid" -> "<sha1> <tree-id>"

    "tree fileid revid" -> "<sha1>"

    "blob fileid revid" -> "<sha1>"

    """

    TDB_MAP_VERSION = 3

    TDB_HASH_SIZE = 50000

    def __init__(self, path=None):

        import tdb

        self.path = path

        if path is None:

            self.db = {}

        else:

            if path not in mapdbs():

                mapdbs()[path] = tdb.Tdb(path, self.TDB_HASH_SIZE, tdb.DEFAULT,

                                          os.O_RDWR|os.O_CREAT)

            self.db = mapdbs()[path]

        try:

            if int(self.db[b"version"]) not in (2, 3):

                trace.warning("SHA Map is incompatible (%s -> %d), rebuilding database.",

                              self.db[b"version"], self.TDB_MAP_VERSION)

                self.db.clear()

        except KeyError:

            pass

        self.db[b"version"] = b'%d' % self.TDB_MAP_VERSION