/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/git/object_store.py

  • Committer: Jelmer Vernooij
  • Date: 2018-11-16 23:15:15 UTC
  • mfrom: (7180 work)
  • mto: This revision was merged to the branch mainline in revision 7183.
  • Revision ID: jelmer@jelmer.uk-20181116231515-zqd2yn6kj8lfydyp
Merge trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/git/object_store.py
41
41
    trace,
42
42
    osutils,
43
43
    ui,
44
 
    urlutils,
45
44
    )
46
45
from ..lock import LogicalLockResult
47
46
from ..revision import (
48
47
    NULL_REVISION,
49
48
    )
50
49
from ..sixish import viewitems
51
 
from ..testament import(
 
50
from ..testament import (
52
51
    StrictTestament3,
53
52
    )
54
53
 
97
96
                inv = tree.inventory
98
97
            return len(inv) * 250
99
98
        self.repository = repository
100
 
        self._cache = lru_cache.LRUSizeCache(max_size=MAX_TREE_CACHE_SIZE,
101
 
            after_cleanup_size=None, compute_size=approx_tree_size)
 
99
        self._cache = lru_cache.LRUSizeCache(
 
100
            max_size=MAX_TREE_CACHE_SIZE, after_cleanup_size=None,
 
101
            compute_size=approx_tree_size)
102
102
 
103
103
    def revision_tree(self, revid):
104
104
        try:
119
119
            else:
120
120
                if tree.get_revision_id() != revid:
121
121
                    raise AssertionError(
122
 
                            "revision id did not match: %s != %s" % (
123
 
                                tree.get_revision_id(), revid))
 
122
                        "revision id did not match: %s != %s" % (
 
123
                            tree.get_revision_id(), revid))
124
124
                trees[revid] = tree
125
125
        for tree in self.repository.revision_trees(todo):
126
126
            trees[tree.get_revision_id()] = tree
165
165
    if len(expected_sha) == 40:
166
166
        if expected_sha != object.sha().hexdigest().encode('ascii'):
167
167
            raise AssertionError("Invalid sha for %r: %s" % (object,
168
 
                expected_sha))
 
168
                                                             expected_sha))
169
169
    elif len(expected_sha) == 20:
170
170
        if expected_sha != object.sha().digest():
171
 
            raise AssertionError("Invalid sha for %r: %s" % (object,
172
 
                sha_to_hex(expected_sha)))
 
171
            raise AssertionError("Invalid sha for %r: %s" % (
 
172
                object, sha_to_hex(expected_sha)))
173
173
    else:
174
174
        raise AssertionError("Unknown length %d for %r" % (len(expected_sha),
175
 
            expected_sha))
176
 
 
177
 
 
178
 
def directory_to_tree(path, children, lookup_ie_sha1, unusual_modes, empty_file_name,
179
 
                      allow_empty=False):
 
175
                                                           expected_sha))
 
176
 
 
177
 
 
178
def directory_to_tree(path, children, lookup_ie_sha1, unusual_modes,
 
179
                      empty_file_name, allow_empty=False):
180
180
    """Create a Git Tree object from a Bazaar directory.
181
181
 
182
182
    :param path: directory path
220
220
    """
221
221
    dirty_dirs = set()
222
222
    new_blobs = []
223
 
    new_contents = {}
224
223
    shamap = {}
225
224
    try:
226
225
        base_tree = parent_trees[0]
228
227
    except IndexError:
229
228
        base_tree = tree._repository.revision_tree(NULL_REVISION)
230
229
        other_parent_trees = []
 
230
 
231
231
    def find_unchanged_parent_ie(file_id, kind, other, parent_trees):
232
232
        for ptree in parent_trees:
233
233
            try:
238
238
                pkind = ptree.kind(ppath, file_id)
239
239
                if kind == "file":
240
240
                    if (pkind == "file" and
241
 
                        ptree.get_file_sha1(ppath, file_id) == other):
242
 
                        return (file_id, ptree.get_file_revision(ppath, file_id))
 
241
                            ptree.get_file_sha1(ppath, file_id) == other):
 
242
                        return (
 
243
                            file_id, ptree.get_file_revision(ppath, file_id))
243
244
                if kind == "symlink":
244
245
                    if (pkind == "symlink" and
245
 
                        ptree.get_symlink_target(ppath, file_id) == other):
246
 
                        return (file_id, ptree.get_file_revision(ppath, file_id))
 
246
                            ptree.get_symlink_target(ppath, file_id) == other):
 
247
                        return (
 
248
                            file_id, ptree.get_file_revision(ppath, file_id))
247
249
        raise KeyError
248
250
 
249
251
    # Find all the changed blobs
255
257
            sha1 = tree.get_file_sha1(path[1], file_id)
256
258
            blob_id = None
257
259
            try:
258
 
                (pfile_id, prevision) = find_unchanged_parent_ie(file_id, kind[1], sha1, other_parent_trees)
 
260
                (pfile_id, prevision) = find_unchanged_parent_ie(
 
261
                    file_id, kind[1], sha1, other_parent_trees)
259
262
            except KeyError:
260
263
                pass
261
264
            else:
275
278
            else:
276
279
                shamap[path[1]] = blob_id
277
280
                if add_cache_entry is not None:
278
 
                    add_cache_entry(("blob", blob_id), (file_id, tree.get_file_revision(path[1])), path[1])
 
281
                    add_cache_entry(
 
282
                        ("blob", blob_id),
 
283
                        (file_id, tree.get_file_revision(path[1])), path[1])
279
284
        elif kind[1] == "symlink":
280
285
            target = tree.get_symlink_target(path[1], file_id)
281
286
            blob = symlink_to_blob(target)
282
287
            shamap[path[1]] = blob.id
283
288
            if add_cache_entry is not None:
284
 
                add_cache_entry(blob, (file_id, tree.get_file_revision(path[1])), path[1])
 
289
                add_cache_entry(
 
290
                    blob, (file_id, tree.get_file_revision(path[1])), path[1])
285
291
            try:
286
 
                find_unchanged_parent_ie(file_id, kind[1], target, other_parent_trees)
 
292
                find_unchanged_parent_ie(
 
293
                    file_id, kind[1], target, other_parent_trees)
287
294
            except KeyError:
288
295
                if changed_content:
289
 
                    yield path[1], blob, (file_id, tree.get_file_revision(path[1], file_id))
 
296
                    yield (path[1], blob,
 
297
                           (file_id, tree.get_file_revision(path[1], file_id)))
290
298
        elif kind[1] is None:
291
299
            shamap[path[1]] = None
292
300
        elif kind[1] != 'directory':
298
306
 
299
307
    # Fetch contents of the blobs that were changed
300
308
    for (path, file_id), chunks in tree.iter_files_bytes(
301
 
        [(path, (path, file_id)) for (path, file_id) in new_blobs]):
 
309
            [(path, (path, file_id)) for (path, file_id) in new_blobs]):
302
310
        obj = Blob()
303
311
        obj.chunked = chunks
304
312
        if add_cache_entry is not None:
305
313
            add_cache_entry(obj, (file_id, tree.get_file_revision(path)), path)
306
 
        yield path, obj, (file_id, tree.get_file_revision(path, file_id))
 
314
        yield path, obj, (file_id, tree.get_file_revision(path))
307
315
        shamap[path] = obj.id
308
316
 
309
317
    for path in unusual_modes:
337
345
        elif ie.kind == "directory":
338
346
            # Not all cache backends store the tree information,
339
347
            # calculate again from scratch
340
 
            ret = directory_to_tree(path, ie.children.values(), ie_to_hexsha,
341
 
                unusual_modes, dummy_file_name, ie.parent_id is None)
 
348
            ret = directory_to_tree(
 
349
                path, ie.children.values(), ie_to_hexsha, unusual_modes,
 
350
                dummy_file_name, ie.parent_id is None)
342
351
            if ret is None:
343
352
                return ret
344
353
            return ret.id
424
433
        if self._map_updated:
425
434
            return
426
435
        if (stop_revision is not None and
427
 
            not self._missing_revisions([stop_revision])):
 
436
                not self._missing_revisions([stop_revision])):
428
437
            return
429
438
        graph = self.repository.get_graph()
430
439
        if stop_revision is None:
451
460
        try:
452
461
            pb = ui.ui_factory.nested_progress_bar()
453
462
            try:
454
 
                for i, revid in enumerate(graph.iter_topo_order(missing_revids)):
 
463
                for i, revid in enumerate(graph.iter_topo_order(
 
464
                        missing_revids)):
455
465
                    trace.mutter('processing %r', revid)
456
466
                    pb.update("updating git map", i, len(missing_revids))
457
467
                    self._update_sha_map_revision(revid)
459
469
                pb.finished()
460
470
            if stop_revision is None:
461
471
                self._map_updated = True
462
 
        except:
 
472
        except BaseException:
463
473
            self.abort_write_group()
464
474
            raise
465
475
        else:
484
494
            except errors.NoSuchRevision:
485
495
                return None
486
496
        return self.mapping.export_commit(rev, tree_sha, parent_lookup,
487
 
            lossy, verifiers)
 
497
                                          lossy, verifiers)
488
498
 
489
499
    def _create_fileid_map_blob(self, tree):
490
500
        # FIXME: This can probably be a lot more efficient,
507
517
        parent_trees = self.tree_cache.revision_trees(
508
518
            [p for p in rev.parent_ids if p in present_parents])
509
519
        root_tree = None
510
 
        for path, obj, bzr_key_data in _tree_to_objects(tree, parent_trees,
511
 
                self._cache.idmap, unusual_modes,
 
520
        for path, obj, bzr_key_data in _tree_to_objects(
 
521
                tree, parent_trees, self._cache.idmap, unusual_modes,
512
522
                self.mapping.BZR_DUMMY_FILE, add_cache_entry):
513
523
            if path == "":
514
524
                root_tree = obj
535
545
        yield "", root_tree
536
546
        if not lossy:
537
547
            testament3 = StrictTestament3(rev, tree)
538
 
            verifiers = { "testament3-sha1": testament3.as_sha1() }
 
548
            verifiers = {"testament3-sha1": testament3.as_sha1()}
539
549
        else:
540
550
            verifiers = {}
541
551
        commit_obj = self._reconstruct_commit(rev, root_tree.id,
542
 
            lossy=lossy, verifiers=verifiers)
 
552
                                              lossy=lossy, verifiers=verifiers)
543
553
        try:
544
554
            foreign_revid, mapping = mapping_registry.parse_revision_id(
545
555
                rev.revision_id)
584
594
                tree = self.tree_cache.revision_tree(revision)
585
595
                path = tree.id2path(file_id)
586
596
                if tree.kind(path, file_id) == 'symlink':
587
 
                    blob = symlink_to_blob(tree.get_symlink_target(path, file_id))
 
597
                    blob = symlink_to_blob(
 
598
                        tree.get_symlink_target(path, file_id))
588
599
            _check_expected_sha(expected_sha, blob)
589
600
            yield blob
590
601
 
591
602
    def _reconstruct_tree(self, fileid, revid, bzr_tree, unusual_modes,
592
 
        expected_sha=None):
 
603
                          expected_sha=None):
593
604
        """Return a Git Tree object from a file id and a revision stored in bzr.
594
605
 
595
606
        :param fileid: fileid in the tree.
599
610
            if entry.kind == "directory":
600
611
                try:
601
612
                    return self._cache.idmap.lookup_tree_id(entry.file_id,
602
 
                        revid)
 
613
                                                            revid)
603
614
                except (NotImplementedError, KeyError):
604
 
                    obj = self._reconstruct_tree(entry.file_id, revid, bzr_tree,
605
 
                        unusual_modes)
 
615
                    obj = self._reconstruct_tree(
 
616
                        entry.file_id, revid, bzr_tree, unusual_modes)
606
617
                    if obj is None:
607
618
                        return None
608
619
                    else:
610
621
            elif entry.kind in ("file", "symlink"):
611
622
                try:
612
623
                    return self._cache.idmap.lookup_blob_id(entry.file_id,
613
 
                        entry.revision)
 
624
                                                            entry.revision)
614
625
                except KeyError:
615
626
                    # no-change merge?
616
627
                    return next(self._reconstruct_blobs(
622
633
                raise AssertionError("unknown entry kind '%s'" % entry.kind)
623
634
        path = bzr_tree.id2path(fileid)
624
635
        tree = directory_to_tree(
625
 
                path,
626
 
                bzr_tree.iter_child_entries(path),
627
 
                get_ie_sha1, unusual_modes, self.mapping.BZR_DUMMY_FILE,
628
 
                bzr_tree.get_root_id() == fileid)
 
636
            path,
 
637
            bzr_tree.iter_child_entries(path),
 
638
            get_ie_sha1, unusual_modes, self.mapping.BZR_DUMMY_FILE,
 
639
            bzr_tree.get_root_id() == fileid)
629
640
        if (bzr_tree.get_root_id() == fileid and
630
 
            self.mapping.BZR_FILE_IDS_FILE is not None):
 
641
                self.mapping.BZR_FILE_IDS_FILE is not None):
631
642
            if tree is None:
632
643
                tree = Tree()
633
644
            b = self._create_fileid_map_blob(bzr_tree)
744
755
                    trace.mutter('entry for %s %s in shamap: %r, but not '
745
756
                                 'found in repository', kind, sha, type_data)
746
757
                    raise KeyError(sha)
747
 
                # FIXME: the type data should say whether conversion was lossless
748
 
                commit = self._reconstruct_commit(rev, tree_sha,
749
 
                    lossy=(not self.mapping.roundtripping), verifiers=verifiers)
 
758
                # FIXME: the type data should say whether conversion was
 
759
                # lossless
 
760
                commit = self._reconstruct_commit(
 
761
                    rev, tree_sha, lossy=(not self.mapping.roundtripping),
 
762
                    verifiers=verifiers)
750
763
                _check_expected_sha(sha, commit)
751
764
                return commit
752
765
            elif kind == "blob":
759
772
                    tree = self.tree_cache.revision_tree(revid)
760
773
                    rev = self.repository.get_revision(revid)
761
774
                except errors.NoSuchRevision:
762
 
                    trace.mutter('entry for %s %s in shamap: %r, but not found in '
 
775
                    trace.mutter(
 
776
                        'entry for %s %s in shamap: %r, but not found in '
763
777
                        'repository', kind, sha, type_data)
764
778
                    raise KeyError(sha)
765
779
                unusual_modes = extract_unusual_modes(rev)
766
780
                try:
767
 
                    return self._reconstruct_tree(fileid, revid,
768
 
                        tree, unusual_modes, expected_sha=sha)
 
781
                    return self._reconstruct_tree(
 
782
                        fileid, revid, tree, unusual_modes, expected_sha=sha)
769
783
                except errors.NoSuchRevision:
770
784
                    raise KeyError(sha)
771
785
            else:
774
788
            raise KeyError(sha)
775
789
 
776
790
    def generate_lossy_pack_data(self, have, want, progress=None,
777
 
            get_tagged=None, ofs_delta=False):
 
791
                                 get_tagged=None, ofs_delta=False):
778
792
        return pack_objects_to_data(
779
 
                self.generate_pack_contents(have, want, progress, get_tagged,
780
 
            lossy=True))
 
793
            self.generate_pack_contents(have, want, progress, get_tagged,
 
794
                                        lossy=True))
781
795
 
782
796
    def generate_pack_contents(self, have, want, progress=None,
783
 
            ofs_delta=False, get_tagged=None, lossy=False):
 
797
                               ofs_delta=False, get_tagged=None, lossy=False):
784
798
        """Iterate over the contents of a pack file.
785
799
 
786
800
        :param have: List of SHA1s of objects that should not be sent
833
847
        import os
834
848
        fd, path = tempfile.mkstemp(suffix=".pack")
835
849
        f = os.fdopen(fd, 'wb')
 
850
 
836
851
        def commit():
837
852
            from .fetch import import_git_objects
838
853
            os.fsync(fd)
840
855
            if os.path.getsize(path) == 0:
841
856
                return
842
857
            pd = PackData(path)
843
 
            pd.create_index_v2(path[:-5]+".idx", self.object_store.get_raw)
 
858
            pd.create_index_v2(path[:-5] + ".idx", self.object_store.get_raw)
844
859
 
845
860
            p = Pack(path[:-5])
846
861
            with self.repository.lock_write():
847
862
                self.repository.start_write_group()
848
863
                try:
849
864
                    import_git_objects(self.repository, self.mapping,
850
 
                        p.iterobjects(get_raw=self.get_raw),
851
 
                        self.object_store)
852
 
                except:
 
865
                                       p.iterobjects(get_raw=self.get_raw),
 
866
                                       self.object_store)
 
867
                except BaseException:
853
868
                    self.repository.abort_write_group()
854
869
                    raise
855
870
                else: