/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/git/transportgit.py

  • Committer: Jelmer Vernooij
  • Date: 2019-03-04 00:16:27 UTC
  • mfrom: (7293 work)
  • mto: This revision was merged to the branch mainline in revision 7318.
  • Revision ID: jelmer@jelmer.uk-20190304001627-v6u7o6pf97tukhek
Merge trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/git/transportgit.py
24
24
import sys
25
25
 
26
26
from dulwich.errors import (
27
 
    NotGitRepository,
28
27
    NoIndexPresent,
29
28
    )
30
29
from dulwich.file import (
37
36
from dulwich.object_store import (
38
37
    PackBasedObjectStore,
39
38
    PACKDIR,
 
39
    read_packs_file,
40
40
    )
41
41
from dulwich.pack import (
42
42
    MemoryPackIndex,
56
56
    CONTROLDIR,
57
57
    INDEX_FILENAME,
58
58
    OBJECTDIR,
59
 
    REFSDIR,
60
59
    SYMREF,
61
60
    check_ref_format,
62
61
    read_packed_refs_with_peeled,
69
68
    transport as _mod_transport,
70
69
    urlutils,
71
70
    )
72
 
from ..sixish import (
73
 
    PY3,
74
 
    text_type,
75
 
    )
76
71
from ..errors import (
77
72
    AlreadyControlDirError,
78
73
    FileExists,
79
74
    LockBroken,
80
 
    LockError,
81
75
    LockContention,
82
76
    NotLocalUrl,
83
77
    NoSuchFile,
104
98
 
105
99
    def _ensure_dir_exists(self, path):
106
100
        for n in range(path.count("/")):
107
 
            dirname = "/".join(path.split("/")[:n+1])
 
101
            dirname = "/".join(path.split("/")[:n + 1])
108
102
            try:
109
103
                self.transport.mkdir(dirname)
110
104
            except FileExists:
133
127
        else:
134
128
            keys.add(b"HEAD")
135
129
        try:
136
 
            iter_files = list(self.transport.clone("refs").iter_files_recursive())
 
130
            iter_files = list(self.transport.clone(
 
131
                "refs").iter_files_recursive())
137
132
            for filename in iter_files:
138
133
                unquoted_filename = urlutils.unquote_to_bytes(filename)
139
134
                refname = osutils.pathjoin(b"refs", unquoted_filename)
164
159
                return {}
165
160
            try:
166
161
                first_line = next(iter(f)).rstrip()
167
 
                if (first_line.startswith("# pack-refs") and " peeled" in
 
162
                if (first_line.startswith(b"# pack-refs") and b" peeled" in
168
163
                        first_line):
169
164
                    for sha, name, peeled in read_packed_refs_with_peeled(f):
170
165
                        self._packed_refs[name] = sha
182
177
        """Return the cached peeled value of a ref, if available.
183
178
 
184
179
        :param name: Name of the ref to peel
185
 
        :return: The peeled value of the ref. If the ref is known not point to a
186
 
            tag, this will be the SHA the ref refers to. If the ref may point to
187
 
            a tag, but no cached information is available, None is returned.
 
180
        :return: The peeled value of the ref. If the ref is known not point to
 
181
            a tag, this will be the SHA the ref refers to. If the ref may point
 
182
            to a tag, but no cached information is available, None is returned.
188
183
        """
189
184
        self.get_packed_refs()
190
185
        if self._peeled_refs is None or name not in self._packed_refs:
222
217
                return header + next(iter(f)).rstrip(b"\r\n")
223
218
            else:
224
219
                # Read only the first 40 bytes
225
 
                return header + f.read(40-len(SYMREF))
 
220
                return header + f.read(40 - len(SYMREF))
226
221
 
227
222
    def _remove_packed_ref(self, name):
228
223
        if self._packed_refs is None:
257
252
            self._ensure_dir_exists(urlutils.quote_from_bytes(name))
258
253
        else:
259
254
            transport = self.worktree_transport
260
 
        transport.put_bytes(urlutils.quote_from_bytes(name), SYMREF + other + b'\n')
 
255
        transport.put_bytes(urlutils.quote_from_bytes(
 
256
            name), SYMREF + other + b'\n')
261
257
 
262
258
    def set_if_equals(self, name, old_ref, new_ref):
263
259
        """Set a refname to new_ref only if it currently equals old_ref.
281
277
        else:
282
278
            transport = self.transport
283
279
            self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
284
 
        transport.put_bytes(urlutils.quote_from_bytes(realname), new_ref+b"\n")
 
280
        transport.put_bytes(urlutils.quote_from_bytes(
 
281
            realname), new_ref + b"\n")
285
282
        return True
286
283
 
287
284
    def add_if_new(self, name, ref):
307
304
        else:
308
305
            transport = self.transport
309
306
            self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
310
 
        transport.put_bytes(urlutils.quote_from_bytes(realname), ref+b"\n")
 
307
        transport.put_bytes(urlutils.quote_from_bytes(realname), ref + b"\n")
311
308
        return True
312
309
 
313
310
    def remove_if_equals(self, name, old_ref):
317
314
        perform an atomic compare-and-delete operation.
318
315
 
319
316
        :param name: The refname to delete.
320
 
        :param old_ref: The old sha the refname must refer to, or None to delete
321
 
            unconditionally.
 
317
        :param old_ref: The old sha the refname must refer to, or None to
 
318
            delete unconditionally.
322
319
        :return: True if the delete was successful, False otherwise.
323
320
        """
324
321
        self._check_refname(name)
347
344
            transport = self.transport
348
345
        lockname = name + b".lock"
349
346
        try:
350
 
            self.transport.delete(urlutils.quote_from_bytes(lockname))
 
347
            transport.delete(urlutils.quote_from_bytes(lockname))
351
348
        except NoSuchFile:
352
349
            pass
353
350
 
359
356
        self._ensure_dir_exists(urlutils.quote_from_bytes(name))
360
357
        lockname = urlutils.quote_from_bytes(name + b".lock")
361
358
        try:
362
 
            local_path = self.transport.local_abspath(urlutils.quote_from_bytes(name))
 
359
            local_path = transport.local_abspath(
 
360
                urlutils.quote_from_bytes(name))
363
361
        except NotLocalUrl:
364
362
            # This is racy, but what can we do?
365
 
            if self.transport.has(lockname):
 
363
            if transport.has(lockname):
366
364
                raise LockContention(name)
367
 
            lock_result = self.transport.put_bytes(lockname, b'Locked by brz-git')
368
 
            return LogicalLockResult(lambda: self.transport.delete(lockname))
 
365
            transport.put_bytes(lockname, b'Locked by brz-git')
 
366
            return LogicalLockResult(lambda: transport.delete(lockname))
369
367
        else:
370
368
            try:
371
369
                gf = GitFile(local_path, 'wb')
374
372
            else:
375
373
                def unlock():
376
374
                    try:
377
 
                        self.transport.delete(lockname)
 
375
                        transport.delete(lockname)
378
376
                    except NoSuchFile:
379
377
                        raise LockBroken(lockname)
380
 
                    # GitFile.abort doesn't care if the lock has already disappeared
 
378
                    # GitFile.abort doesn't care if the lock has already
 
379
                    # disappeared
381
380
                    gf.abort()
382
381
                return LogicalLockResult(unlock)
383
382
 
412
411
            else:
413
412
                self._controltransport = self.transport.clone('.git')
414
413
        else:
415
 
            self._controltransport = self.transport.clone(urlutils.quote_from_bytes(path))
 
414
            self._controltransport = self.transport.clone(
 
415
                urlutils.quote_from_bytes(path))
416
416
        commondir = self.get_named_file(COMMONDIR)
417
417
        if commondir is not None:
418
418
            with commondir:
429
429
        if refs_text is not None:
430
430
            refs_container = InfoRefsContainer(BytesIO(refs_text))
431
431
            try:
432
 
                head = TransportRefsContainer(self._commontransport).read_loose_ref("HEAD")
 
432
                head = TransportRefsContainer(
 
433
                    self._commontransport).read_loose_ref("HEAD")
433
434
            except KeyError:
434
435
                pass
435
436
            else:
436
437
                refs_container._refs["HEAD"] = head
437
438
        else:
438
439
            refs_container = TransportRefsContainer(
439
 
                    self._commontransport, self._controltransport)
 
440
                self._commontransport, self._controltransport)
440
441
        super(TransportRepo, self).__init__(object_store,
441
 
                refs_container)
 
442
                                            refs_container)
442
443
 
443
444
    def controldir(self):
444
445
        return self._controltransport.local_abspath('.')
587
588
                ret.append(l)
588
589
            return ret
589
590
 
590
 
    @property
591
 
    def packs(self):
592
 
        # FIXME: Never invalidates.
593
 
        if not self._pack_cache:
594
 
            self._update_pack_cache()
595
 
        return self._pack_cache.values()
596
 
 
597
591
    def _update_pack_cache(self):
598
 
        for pack in self._load_packs():
599
 
            self._pack_cache[pack._basename] = pack
 
592
        pack_files = set()
 
593
        pack_dir_contents = self._pack_names()
 
594
        for name in pack_dir_contents:
 
595
            if name.startswith("pack-") and name.endswith(".pack"):
 
596
                # verify that idx exists first (otherwise the pack was not yet
 
597
                # fully written)
 
598
                idx_name = os.path.splitext(name)[0] + ".idx"
 
599
                if idx_name in pack_dir_contents:
 
600
                    pack_files.add(os.path.splitext(name)[0])
 
601
 
 
602
        new_packs = []
 
603
        for basename in pack_files:
 
604
            pack_name = basename + ".pack"
 
605
            if basename not in self._pack_cache:
 
606
                try:
 
607
                    size = self.pack_transport.stat(pack_name).st_size
 
608
                except TransportNotPossible:
 
609
                    f = self.pack_transport.get(pack_name)
 
610
                    pd = PackData(pack_name, f)
 
611
                else:
 
612
                    pd = PackData(
 
613
                        pack_name, self.pack_transport.get(pack_name),
 
614
                        size=size)
 
615
                idxname = basename + ".idx"
 
616
                idx = load_pack_index_file(
 
617
                    idxname, self.pack_transport.get(idxname))
 
618
                pack = Pack.from_objects(pd, idx)
 
619
                pack._basename = basename
 
620
                self._pack_cache[basename] = pack
 
621
                new_packs.append(pack)
 
622
        # Remove disappeared pack files
 
623
        for f in set(self._pack_cache) - pack_files:
 
624
            self._pack_cache.pop(f).close()
 
625
        return new_packs
600
626
 
601
627
    def _pack_names(self):
602
628
        try:
603
 
            f = self.transport.get('info/packs')
604
 
        except NoSuchFile:
605
629
            return self.pack_transport.list_dir(".")
606
 
        else:
607
 
            with f:
608
 
                ret = []
609
 
                for line in f.read().splitlines():
610
 
                    if not line:
611
 
                        continue
612
 
                    (kind, name) = line.split(b" ", 1)
613
 
                    if kind != b"P":
614
 
                        continue
615
 
                    ret.append(name)
616
 
                return ret
 
630
        except TransportNotPossible:
 
631
            try:
 
632
                f = self.transport.get('info/packs')
 
633
            except NoSuchFile:
 
634
                # Hmm, warn about running 'git update-server-info' ?
 
635
                return iter([])
 
636
            else:
 
637
                with f:
 
638
                    return read_packs_file(f)
 
639
        except NoSuchFile:
 
640
            return iter([])
617
641
 
618
642
    def _remove_pack(self, pack):
619
643
        self.pack_transport.delete(os.path.basename(pack.index.path))
620
644
        self.pack_transport.delete(pack.data.filename)
621
 
 
622
 
    def _load_packs(self):
623
 
        ret = []
624
 
        for name in self._pack_names():
625
 
            if name.startswith("pack-") and name.endswith(".pack"):
626
 
                try:
627
 
                    size = self.pack_transport.stat(name).st_size
628
 
                except TransportNotPossible:
629
 
                    f = self.pack_transport.get(name)
630
 
                    pd = PackData(name, f, size=len(contents))
631
 
                else:
632
 
                    pd = PackData(name, self.pack_transport.get(name),
633
 
                            size=size)
634
 
                idxname = name.replace(".pack", ".idx")
635
 
                idx = load_pack_index_file(idxname, self.pack_transport.get(idxname))
636
 
                pack = Pack.from_objects(pd, idx)
637
 
                pack._basename = idxname[:-4]
638
 
                ret.append(pack)
639
 
        return ret
 
645
        try:
 
646
            del self._pack_cache[os.path.basename(pack._basename)]
 
647
        except KeyError:
 
648
            pass
640
649
 
641
650
    def _iter_loose_objects(self):
642
651
        for base in self.transport.list_dir('.'):
643
652
            if len(base) != 2:
644
653
                continue
645
654
            for rest in self.transport.list_dir(base):
646
 
                yield (base+rest).encode(sys.getfilesystemencoding())
 
655
                yield (base + rest).encode(sys.getfilesystemencoding())
647
656
 
648
657
    def _split_loose_object(self, sha):
649
658
        return (sha[:2], sha[2:])
672
681
            pass
673
682
        path = urlutils.quote_from_bytes(osutils.pathjoin(dir, file))
674
683
        if self.transport.has(path):
675
 
            return # Already there, no need to write again
 
684
            return  # Already there, no need to write again
676
685
        self.transport.put_bytes(path, obj.as_legacy_object())
677
686
 
678
687
    def move_in_pack(self, f):
686
695
        f.seek(0)
687
696
        p = PackData("", f, len(f.getvalue()))
688
697
        entries = p.sorted_entries()
689
 
        basename = "pack-%s" % iter_sha1(entry[0] for entry in entries).decode('ascii')
 
698
        basename = "pack-%s" % iter_sha1(entry[0]
 
699
                                         for entry in entries).decode('ascii')
690
700
        p._filename = basename + ".pack"
691
701
        f.seek(0)
692
702
        self.pack_transport.put_file(basename + ".pack", f)
696
706
        finally:
697
707
            idxfile.close()
698
708
        idxfile = self.pack_transport.get(basename + ".idx")
699
 
        idx = load_pack_index_file(basename+".idx", idxfile)
 
709
        idx = load_pack_index_file(basename + ".idx", idxfile)
700
710
        final_pack = Pack.from_objects(p, idx)
701
711
        final_pack._basename = basename
702
 
        self._add_known_pack(basename, final_pack)
 
712
        self._add_cached_pack(basename, final_pack)
703
713
        return final_pack
704
714
 
705
715
    def move_in_thin_pack(self, f):
714
724
        p = Pack('', resolve_ext_ref=self.get_raw)
715
725
        p._data = PackData.from_file(f, len(f.getvalue()))
716
726
        p._data.pack = p
717
 
        p._idx_load = lambda: MemoryPackIndex(p.data.sorted_entries(), p.data.get_stored_checksum())
 
727
        p._idx_load = lambda: MemoryPackIndex(
 
728
            p.data.sorted_entries(), p.data.get_stored_checksum())
718
729
 
719
730
        pack_sha = p.index.objects_sha1()
720
731
 
721
732
        datafile = self.pack_transport.open_write_stream(
722
 
                "pack-%s.pack" % pack_sha.decode('ascii'))
 
733
            "pack-%s.pack" % pack_sha.decode('ascii'))
723
734
        try:
724
735
            entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
725
736
        finally:
731
742
            write_pack_index_v2(idxfile, entries, data_sum)
732
743
        finally:
733
744
            idxfile.close()
734
 
        # TODO(jelmer): Just add new pack to the cache
735
 
        self._flush_pack_cache()
736
745
 
737
746
    def add_pack(self):
738
747
        """Add a new pack to this object store.
741
750
            call when the pack is finished.
742
751
        """
743
752
        f = BytesIO()
 
753
 
744
754
        def commit():
745
755
            if len(f.getvalue()) > 0:
746
756
                return self.move_in_pack(f)
747
757
            else:
748
758
                return None
 
759
 
749
760
        def abort():
750
761
            return None
751
762
        return f, commit, abort