/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to fetch.py

Merge thin-pack work.

Show diffs side-by-side

added added

removed removed

Lines of Context:
fetch.py
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
16
 
17
17
from bzrlib import osutils, ui, urlutils
18
 
from bzrlib.errors import InvalidRevisionId
 
18
from bzrlib.errors import InvalidRevisionId, NoSuchRevision
19
19
from bzrlib.inventory import Inventory
20
20
from bzrlib.repository import InterRepository
21
21
from bzrlib.trace import info
22
22
from bzrlib.tsort import topo_sort
23
23
 
24
 
from bzrlib.plugins.git import git
25
24
from bzrlib.plugins.git.repository import (
26
25
        LocalGitRepository, 
27
26
        GitRepository, 
28
27
        GitFormat,
29
28
        )
 
29
from bzrlib.plugins.git.shamap import GitObjectConverter
30
30
from bzrlib.plugins.git.remote import RemoteGitRepository
31
31
 
 
32
import dulwich as git
 
33
from dulwich.client import SimpleFetchGraphWalker
32
34
from dulwich.objects import Commit
33
35
 
34
36
from cStringIO import StringIO
35
37
 
36
38
 
37
39
class BzrFetchGraphWalker(object):
 
40
    """GraphWalker implementation that uses a Bazaar repository."""
38
41
 
39
42
    def __init__(self, repository, mapping):
40
43
        self.repository = repository
43
46
        self.heads = set(repository.all_revision_ids())
44
47
        self.parents = {}
45
48
 
 
49
    def __iter__(self):
 
50
        return iter(self.next, None)
 
51
 
46
52
    def ack(self, sha):
47
53
        revid = self.mapping.revision_id_foreign_to_bzr(sha)
48
54
        self.remove(revid)
49
55
 
50
56
    def remove(self, revid):
51
57
        self.done.add(revid)
52
 
        if ref in self.heads:
 
58
        if revid in self.heads:
53
59
            self.heads.remove(revid)
54
60
        if revid in self.parents:
55
61
            for p in self.parents[revid]:
63
69
            self.heads.update([p for p in ps if not p in self.done])
64
70
            try:
65
71
                self.done.add(ret)
66
 
                return self.mapping.revision_id_bzr_to_foreign(ret)
 
72
                return self.mapping.revision_id_bzr_to_foreign(ret)[0]
67
73
            except InvalidRevisionId:
68
74
                pass
69
75
        return None
121
127
            raise AssertionError("Unknown blob kind, perms=%r." % (mode,))
122
128
 
123
129
 
124
 
def import_git_objects(repo, mapping, object_iter, pb=None):
 
130
def import_git_objects(repo, mapping, object_iter, target_git_object_retriever, 
 
131
        pb=None):
125
132
    """Import a set of git objects into a bzr repository.
126
133
 
127
134
    :param repo: Bazaar repository
129
136
    :param object_iter: Iterator over Git objects.
130
137
    """
131
138
    # TODO: a more (memory-)efficient implementation of this
132
 
    objects = {}
133
 
    for i, o in enumerate(object_iter):
134
 
        if pb is not None:
135
 
            pb.update("fetching objects", i) 
136
 
        objects[o.id] = o
137
139
    graph = []
138
140
    root_trees = {}
139
141
    revisions = {}
140
142
    # Find and convert commit objects
141
 
    for o in objects.itervalues():
 
143
    for o in object_iter.iterobjects():
142
144
        if isinstance(o, Commit):
143
145
            rev = mapping.import_commit(o)
144
 
            root_trees[rev.revision_id] = objects[o.tree]
 
146
            root_trees[rev.revision_id] = object_iter[o.tree]
145
147
            revisions[rev.revision_id] = rev
146
148
            graph.append((rev.revision_id, rev.parent_ids))
147
149
    # Order the revisions
157
159
        inv = Inventory()
158
160
        inv.revision_id = rev.revision_id
159
161
        def lookup_object(sha):
160
 
            if sha in objects:
161
 
                return objects[sha]
162
 
            return reconstruct_git_object(repo, mapping, sha)
 
162
            if sha in object_iter:
 
163
                return object_iter[sha]
 
164
            return target_git_object_retriever(sha)
163
165
        parent_invs = [repo.get_inventory(r) for r in rev.parent_ids]
164
 
        import_git_tree(repo, mapping, "", root_tree, inv, parent_invs, lookup_object)
 
166
        import_git_tree(repo, mapping, "", root_tree, inv, parent_invs, 
 
167
            lookup_object)
165
168
        repo.add_revision(rev.revision_id, rev, inv)
166
169
 
167
170
 
168
 
def reconstruct_git_commit(repo, rev):
169
 
    raise NotImplementedError(self.reconstruct_git_commit)
170
 
 
171
 
 
172
171
def reconstruct_git_object(repo, mapping, sha):
173
 
    # Commit
174
 
    revid = mapping.revision_id_foreign_to_bzr(sha)
175
 
    try:
176
 
        rev = repo.get_revision(revid)
177
 
    except NoSuchRevision:
178
 
        pass
179
 
    else:
180
 
        return reconstruct_git_commit(rev)
 
172
    import pdb; pdb.set_trace()
181
173
 
182
174
    # TODO: Tree
183
175
    # TODO: Blob
184
176
    raise KeyError("No such object %s" % sha)
185
177
 
186
178
 
187
 
class InterGitRepository(InterRepository):
 
179
class InterGitNonGitRepository(InterRepository):
188
180
 
189
181
    _matching_repo_format = GitFormat()
190
182
 
196
188
        """See InterRepository.copy_content."""
197
189
        self.fetch(revision_id, pb, find_ghosts=False)
198
190
 
199
 
    def fetch(self, revision_id=None, pb=None, find_ghosts=False, 
200
 
              mapping=None):
201
 
        if mapping is None:
202
 
            mapping = self.source.get_mapping()
 
191
    def fetch_objects(self, determine_wants, mapping, pb=None):
203
192
        def progress(text):
204
 
            pb.note("git: %s", text)
205
 
        def determine_wants(heads):
206
 
            if revision_id is None:
207
 
                ret = heads.values()
208
 
            else:
209
 
                ret = [mapping.revision_id_bzr_to_foreign(revision_id)]
210
 
            return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
 
193
            pb.update("git: %s" % text.rstrip("\r\n"), 0, 0)
211
194
        graph_walker = BzrFetchGraphWalker(self.target, mapping)
212
195
        create_pb = None
213
196
        if pb is None:
214
197
            create_pb = pb = ui.ui_factory.nested_progress_bar()
 
198
        target_git_object_retriever = GitObjectConverter(self.target, mapping)
 
199
        
215
200
        try:
216
201
            self.target.lock_write()
217
202
            try:
218
203
                self.target.start_write_group()
219
204
                try:
220
 
                    import_git_objects(self.target, mapping,
221
 
                        iter(self.source.fetch_objects(determine_wants, graph_walker, 
222
 
                            progress)), pb)
 
205
                    objects_iter = self.source.fetch_objects(determine_wants, 
 
206
                                graph_walker, 
 
207
                                target_git_object_retriever.__getitem__, 
 
208
                                progress)
 
209
                    import_git_objects(self.target, mapping, objects_iter, 
 
210
                            target_git_object_retriever, pb)
223
211
                finally:
224
212
                    self.target.commit_write_group()
225
213
            finally:
228
216
            if create_pb:
229
217
                create_pb.finished()
230
218
 
 
219
    def fetch(self, revision_id=None, pb=None, find_ghosts=False, 
 
220
              mapping=None):
 
221
        if mapping is None:
 
222
            mapping = self.source.get_mapping()
 
223
        def determine_wants(heads):
 
224
            if revision_id is None:
 
225
                ret = heads.values()
 
226
            else:
 
227
                ret = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
 
228
            return [rev for rev in ret if not self.target.has_revision(mapping.revision_id_foreign_to_bzr(rev))]
 
229
        return self.fetch_objects(determine_wants, mapping, pb)
 
230
 
231
231
    @staticmethod
232
232
    def is_compatible(source, target):
233
233
        """Be compatible with GitRepository."""
234
234
        # FIXME: Also check target uses VersionedFile
235
235
        return (isinstance(source, GitRepository) and 
236
 
                target.supports_rich_root())
 
236
                target.supports_rich_root() and
 
237
                not isinstance(target, GitRepository))
 
238
 
 
239
 
 
240
class InterGitRepository(InterRepository):
 
241
 
 
242
    _matching_repo_format = GitFormat()
 
243
 
 
244
    @staticmethod
 
245
    def _get_repo_format_to_test():
 
246
        return None
 
247
 
 
248
    def copy_content(self, revision_id=None, pb=None):
 
249
        """See InterRepository.copy_content."""
 
250
        self.fetch(revision_id, pb, find_ghosts=False)
 
251
 
 
252
    def fetch(self, revision_id=None, pb=None, find_ghosts=False, 
 
253
              mapping=None):
 
254
        if mapping is None:
 
255
            mapping = self.source.get_mapping()
 
256
        def progress(text):
 
257
            info("git: %s", text)
 
258
        r = self.target._git
 
259
        if revision_id is None:
 
260
            determine_wants = lambda x: [y for y in x.values() if not y in r.object_store]
 
261
        else:
 
262
            args = [mapping.revision_id_bzr_to_foreign(revision_id)[0]]
 
263
            determine_wants = lambda x: [y for y in args if not y in r.object_store]
 
264
 
 
265
        graphwalker = SimpleFetchGraphWalker(r.heads().values(), r.get_parents)
 
266
        f, commit = r.object_store.add_pack()
 
267
        try:
 
268
            self.source._git.fetch_pack(path, determine_wants, graphwalker, f.write, progress)
 
269
            f.close()
 
270
            commit()
 
271
        except:
 
272
            f.close()
 
273
            raise
 
274
 
 
275
    @staticmethod
 
276
    def is_compatible(source, target):
 
277
        """Be compatible with GitRepository."""
 
278
        return (isinstance(source, GitRepository) and 
 
279
                isinstance(target, GitRepository))