1
# Copyright (C) 2010-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""A Git repository implementation that uses a Bazaar transport."""
19
from io import BytesIO
24
from dulwich.errors import (
27
from dulwich.file import (
31
from dulwich.objects import (
34
from dulwich.object_store import (
39
from dulwich.pack import (
48
from dulwich.repo import (
59
read_packed_refs_with_peeled,
66
transport as _mod_transport,
69
from ..errors import (
70
AlreadyControlDirError,
80
from ..lock import LogicalLockResult
81
from ..trace import warning
84
class TransportRefsContainer(RefsContainer):
85
"""Refs container that reads refs from a transport."""
87
def __init__(self, transport, worktree_transport=None):
88
self.transport = transport
89
if worktree_transport is None:
90
worktree_transport = transport
91
self.worktree_transport = worktree_transport
92
self._packed_refs = None
93
self._peeled_refs = None
96
return "%s(%r)" % (self.__class__.__name__, self.transport)
98
def _ensure_dir_exists(self, path):
99
for n in range(path.count("/")):
100
dirname = "/".join(path.split("/")[:n + 1])
102
self.transport.mkdir(dirname)
106
def subkeys(self, base):
107
"""Refs present in this container under a base.
109
:param base: The base to return refs under.
110
:return: A set of valid refs in this container under the base; the base
111
prefix is stripped from the ref names returned.
114
base_len = len(base) + 1
115
for refname in self.allkeys():
116
if refname.startswith(base):
117
keys.add(refname[base_len:])
123
self.worktree_transport.get_bytes("HEAD")
129
iter_files = list(self.transport.clone(
130
"refs").iter_files_recursive())
131
for filename in iter_files:
132
unquoted_filename = urlutils.unquote_to_bytes(filename)
133
refname = osutils.pathjoin(b"refs", unquoted_filename)
134
if check_ref_format(refname):
136
except (TransportNotPossible, NoSuchFile):
138
keys.update(self.get_packed_refs())
141
def get_packed_refs(self):
142
"""Get contents of the packed-refs file.
144
:return: Dictionary mapping ref names to SHA1s
146
:note: Will return an empty dictionary when no packed-refs file is
149
# TODO: invalidate the cache on repacking
150
if self._packed_refs is None:
151
# set both to empty because we want _peeled_refs to be
152
# None if and only if _packed_refs is also None.
153
self._packed_refs = {}
154
self._peeled_refs = {}
156
f = self.transport.get("packed-refs")
160
first_line = next(iter(f)).rstrip()
161
if (first_line.startswith(b"# pack-refs") and b" peeled" in
163
for sha, name, peeled in read_packed_refs_with_peeled(f):
164
self._packed_refs[name] = sha
166
self._peeled_refs[name] = peeled
169
for sha, name in read_packed_refs(f):
170
self._packed_refs[name] = sha
173
return self._packed_refs
175
def get_peeled(self, name):
176
"""Return the cached peeled value of a ref, if available.
178
:param name: Name of the ref to peel
179
:return: The peeled value of the ref. If the ref is known not point to
180
a tag, this will be the SHA the ref refers to. If the ref may point
181
to a tag, but no cached information is available, None is returned.
183
self.get_packed_refs()
184
if self._peeled_refs is None or name not in self._packed_refs:
185
# No cache: no peeled refs were read, or this ref is loose
187
if name in self._peeled_refs:
188
return self._peeled_refs[name]
193
def read_loose_ref(self, name):
194
"""Read a reference file and return its contents.
196
If the reference file a symbolic reference, only read the first line of
197
the file. Otherwise, only read the first 40 bytes.
199
:param name: the refname to read, relative to refpath
200
:return: The contents of the ref file, or None if the file does not
202
:raises IOError: if any other error occurs
205
transport = self.worktree_transport
207
transport = self.transport
209
f = transport.get(urlutils.quote_from_bytes(name))
214
header = f.read(len(SYMREF))
216
# probably a directory
219
# Read only the first line
220
return header + next(iter(f)).rstrip(b"\r\n")
222
# Read only the first 40 bytes
223
return header + f.read(40 - len(SYMREF))
225
def _remove_packed_ref(self, name):
226
if self._packed_refs is None:
228
# reread cached refs from disk, while holding the lock
230
self._packed_refs = None
231
self.get_packed_refs()
233
if name not in self._packed_refs:
236
del self._packed_refs[name]
237
if name in self._peeled_refs:
238
del self._peeled_refs[name]
239
with self.transport.open_write_stream("packed-refs") as f:
240
write_packed_refs(f, self._packed_refs, self._peeled_refs)
242
def set_symbolic_ref(self, name, other):
243
"""Make a ref point at another ref.
245
:param name: Name of the ref to set
246
:param other: Name of the ref to point at
248
self._check_refname(name)
249
self._check_refname(other)
251
transport = self.transport
252
self._ensure_dir_exists(urlutils.quote_from_bytes(name))
254
transport = self.worktree_transport
255
transport.put_bytes(urlutils.quote_from_bytes(
256
name), SYMREF + other + b'\n')
258
def set_if_equals(self, name, old_ref, new_ref):
259
"""Set a refname to new_ref only if it currently equals old_ref.
261
This method follows all symbolic references, and can be used to perform
262
an atomic compare-and-swap operation.
264
:param name: The refname to set.
265
:param old_ref: The old sha the refname must refer to, or None to set
267
:param new_ref: The new sha the refname will refer to.
268
:return: True if the set was successful, False otherwise.
271
realnames, _ = self.follow(name)
272
realname = realnames[-1]
273
except (KeyError, IndexError):
275
if realname == b'HEAD':
276
transport = self.worktree_transport
278
transport = self.transport
279
self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
280
transport.put_bytes(urlutils.quote_from_bytes(
281
realname), new_ref + b"\n")
284
def add_if_new(self, name, ref):
285
"""Add a new reference only if it does not already exist.
287
This method follows symrefs, and only ensures that the last ref in the
288
chain does not exist.
290
:param name: The refname to set.
291
:param ref: The new sha the refname will refer to.
292
:return: True if the add was successful, False otherwise.
295
realnames, contents = self.follow(name)
296
if contents is not None:
298
realname = realnames[-1]
299
except (KeyError, IndexError):
301
self._check_refname(realname)
302
if realname == b'HEAD':
303
transport = self.worktree_transport
305
transport = self.transport
306
self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
307
transport.put_bytes(urlutils.quote_from_bytes(realname), ref + b"\n")
310
def remove_if_equals(self, name, old_ref):
311
"""Remove a refname only if it currently equals old_ref.
313
This method does not follow symbolic references. It can be used to
314
perform an atomic compare-and-delete operation.
316
:param name: The refname to delete.
317
:param old_ref: The old sha the refname must refer to, or None to
318
delete unconditionally.
319
:return: True if the delete was successful, False otherwise.
321
self._check_refname(name)
324
transport = self.worktree_transport
326
transport = self.transport
328
transport.delete(urlutils.quote_from_bytes(name))
331
self._remove_packed_ref(name)
334
def get(self, name, default=None):
340
def unlock_ref(self, name):
342
transport = self.worktree_transport
344
transport = self.transport
345
lockname = name + b".lock"
347
transport.delete(urlutils.quote_from_bytes(lockname))
351
def lock_ref(self, name):
353
transport = self.worktree_transport
355
transport = self.transport
356
self._ensure_dir_exists(urlutils.quote_from_bytes(name))
357
lockname = urlutils.quote_from_bytes(name + b".lock")
359
local_path = transport.local_abspath(
360
urlutils.quote_from_bytes(name))
362
# This is racy, but what can we do?
363
if transport.has(lockname):
364
raise LockContention(name)
365
transport.put_bytes(lockname, b'Locked by brz-git')
366
return LogicalLockResult(lambda: transport.delete(lockname))
369
gf = GitFile(local_path, 'wb')
370
except FileLocked as e:
371
raise LockContention(name, e)
375
transport.delete(lockname)
377
raise LockBroken(lockname)
378
# GitFile.abort doesn't care if the lock has already
381
return LogicalLockResult(unlock)
384
# TODO(jelmer): Use upstream read_gitfile; unfortunately that expects strings
385
# rather than bytes..
387
"""Read a ``.git`` file.
389
The first line of the file should start with "gitdir: "
391
:param f: File-like object to read from
395
if not cs.startswith(b"gitdir: "):
396
raise ValueError("Expected file to start with 'gitdir: '")
397
return cs[len(b"gitdir: "):].rstrip(b"\n")
400
class TransportRepo(BaseRepo):
402
def __init__(self, transport, bare, refs_text=None):
403
self.transport = transport
406
with transport.get(CONTROLDIR) as f:
407
path = read_gitfile(f)
408
except (ReadError, NoSuchFile):
410
self._controltransport = self.transport
412
self._controltransport = self.transport.clone('.git')
414
self._controltransport = self.transport.clone(
415
urlutils.quote_from_bytes(path))
416
commondir = self.get_named_file(COMMONDIR)
417
if commondir is not None:
419
commondir = os.path.join(
421
commondir.read().rstrip(b"\r\n").decode(
422
sys.getfilesystemencoding()))
423
self._commontransport = \
424
_mod_transport.get_transport_from_path(commondir)
426
self._commontransport = self._controltransport
427
config = self.get_config()
428
object_store = TransportObjectStore.from_config(
429
self._commontransport.clone(OBJECTDIR),
431
if refs_text is not None:
432
refs_container = InfoRefsContainer(BytesIO(refs_text))
434
head = TransportRefsContainer(
435
self._commontransport).read_loose_ref(b"HEAD")
439
refs_container._refs[b"HEAD"] = head
441
refs_container = TransportRefsContainer(
442
self._commontransport, self._controltransport)
443
super(TransportRepo, self).__init__(object_store,
446
def controldir(self):
447
return self._controltransport.local_abspath('.')
450
return self._commontransport.local_abspath('.')
454
return self.transport.local_abspath('.')
456
def _determine_file_mode(self):
457
# Be consistent with bzr
458
if sys.platform == 'win32':
462
def get_named_file(self, path):
463
"""Get a file from the control dir with a specific name.
465
Although the filename should be interpreted as a filename relative to
466
the control dir in a disk-baked Repo, the object returned need not be
467
pointing to a file in that location.
469
:param path: The path to the file, relative to the control dir.
470
:return: An open file object, or None if the file does not exist.
473
return self._controltransport.get(path.lstrip('/'))
477
def _put_named_file(self, relpath, contents):
478
self._controltransport.put_bytes(relpath, contents)
480
def index_path(self):
481
"""Return the path to the index file."""
482
return self._controltransport.local_abspath(INDEX_FILENAME)
484
def open_index(self):
485
"""Open the index for this repository."""
486
from dulwich.index import Index
487
if not self.has_index():
488
raise NoIndexPresent()
489
return Index(self.index_path())
492
"""Check if an index is present."""
493
# Bare repos must never have index files; non-bare repos may have a
494
# missing index file, which is treated as empty.
497
def get_config(self):
498
from dulwich.config import ConfigFile
500
with self._controltransport.get('config') as f:
501
return ConfigFile.from_file(f)
505
def get_config_stack(self):
506
from dulwich.config import StackedConfig
508
p = self.get_config()
514
backends.extend(StackedConfig.default_backends())
515
return StackedConfig(backends, writable=writable)
517
# Here for compatibility with dulwich < 0.19.17
518
def generate_pack_data(self, have, want, progress=None, ofs_delta=None):
519
"""Generate pack data objects for a set of wants/haves.
522
have: List of SHA1s of objects that should not be sent
523
want: List of SHA1s of objects that should be sent
524
ofs_delta: Whether OFS deltas can be included
525
progress: Optional progress reporting method
527
shallow = self.get_shallow()
529
return self.object_store.generate_pack_data(
530
have, want, shallow=shallow,
531
progress=progress, ofs_delta=ofs_delta)
533
return self.object_store.generate_pack_data(
534
have, want, progress=progress, ofs_delta=ofs_delta)
537
return "<%s for %r>" % (self.__class__.__name__, self.transport)
540
def init(cls, transport, bare=False):
543
transport.mkdir(".git")
545
raise AlreadyControlDirError(transport.base)
546
control_transport = transport.clone(".git")
548
control_transport = transport
549
for d in BASE_DIRECTORIES:
551
control_transport.mkdir("/".join(d))
555
control_transport.mkdir(OBJECTDIR)
557
raise AlreadyControlDirError(transport.base)
558
TransportObjectStore.init(control_transport.clone(OBJECTDIR))
559
ret = cls(transport, bare)
560
ret.refs.set_symbolic_ref(b"HEAD", b"refs/heads/master")
561
ret._init_files(bare)
565
class TransportObjectStore(PackBasedObjectStore):
566
"""Git-style object store that exists on disk."""
568
def __init__(self, transport,
569
loose_compression_level=-1, pack_compression_level=-1):
570
"""Open an object store.
572
:param transport: Transport to open data from
574
super(TransportObjectStore, self).__init__()
575
self.pack_compression_level = pack_compression_level
576
self.loose_compression_level = loose_compression_level
577
self.transport = transport
578
self.pack_transport = self.transport.clone(PACKDIR)
579
self._alternates = None
582
def from_config(cls, path, config):
584
default_compression_level = int(config.get(
585
(b'core', ), b'compression').decode())
587
default_compression_level = -1
589
loose_compression_level = int(config.get(
590
(b'core', ), b'looseCompression').decode())
592
loose_compression_level = default_compression_level
594
pack_compression_level = int(config.get(
595
(b'core', ), 'packCompression').decode())
597
pack_compression_level = default_compression_level
598
return cls(path, loose_compression_level, pack_compression_level)
600
def __eq__(self, other):
601
if not isinstance(other, TransportObjectStore):
603
return self.transport == other.transport
606
return "%s(%r)" % (self.__class__.__name__, self.transport)
609
def alternates(self):
610
if self._alternates is not None:
611
return self._alternates
612
self._alternates = []
613
for path in self._read_alternate_paths():
615
t = _mod_transport.get_transport_from_path(path)
616
self._alternates.append(self.__class__(t))
617
return self._alternates
619
def _read_alternate_paths(self):
621
f = self.transport.get("info/alternates")
626
for l in f.read().splitlines():
634
def _update_pack_cache(self):
635
pack_files = set(self._pack_names())
637
for basename in pack_files:
638
pack_name = basename + ".pack"
639
if basename not in self._pack_cache:
641
size = self.pack_transport.stat(pack_name).st_size
642
except TransportNotPossible:
643
f = self.pack_transport.get(pack_name)
644
# TODO(jelmer): Don't read entire file into memory?
645
f = BytesIO(f.read())
646
pd = PackData(pack_name, f)
649
pack_name, self.pack_transport.get(pack_name),
651
idxname = basename + ".idx"
652
idx = load_pack_index_file(
653
idxname, self.pack_transport.get(idxname))
654
pack = Pack.from_objects(pd, idx)
655
pack._basename = basename
656
self._pack_cache[basename] = pack
657
new_packs.append(pack)
658
# Remove disappeared pack files
659
for f in set(self._pack_cache) - pack_files:
660
self._pack_cache.pop(f).close()
663
def _pack_names(self):
666
dir_contents = self.pack_transport.list_dir(".")
667
for name in dir_contents:
668
if name.startswith("pack-") and name.endswith(".pack"):
669
# verify that idx exists first (otherwise the pack was not yet
671
idx_name = os.path.splitext(name)[0] + ".idx"
672
if idx_name in dir_contents:
673
pack_files.append(os.path.splitext(name)[0])
674
except TransportNotPossible:
676
f = self.transport.get('info/packs')
678
warning('No info/packs on remote host;'
679
'run \'git update-server-info\' on remote.')
683
os.path.splitext(name)[0]
684
for name in read_packs_file(f)]
689
def _remove_pack(self, pack):
690
self.pack_transport.delete(os.path.basename(pack.index.path))
691
self.pack_transport.delete(pack.data.filename)
693
del self._pack_cache[os.path.basename(pack._basename)]
697
def _iter_loose_objects(self):
698
for base in self.transport.list_dir('.'):
701
for rest in self.transport.list_dir(base):
702
yield (base + rest).encode(sys.getfilesystemencoding())
704
def _split_loose_object(self, sha):
705
return (sha[:2], sha[2:])
707
def _remove_loose_object(self, sha):
708
path = osutils.joinpath(self._split_loose_object(sha))
709
self.transport.delete(urlutils.quote_from_bytes(path))
711
def _get_loose_object(self, sha):
712
path = osutils.joinpath(self._split_loose_object(sha))
714
with self.transport.get(urlutils.quote_from_bytes(path)) as f:
715
return ShaFile.from_file(f)
719
def add_object(self, obj):
720
"""Add a single object to this object store.
722
:param obj: Object to add
724
(dir, file) = self._split_loose_object(obj.id)
726
self.transport.mkdir(urlutils.quote_from_bytes(dir))
729
path = urlutils.quote_from_bytes(osutils.pathjoin(dir, file))
730
if self.transport.has(path):
731
return # Already there, no need to write again
732
# Backwards compatibility with Dulwich < 0.20, which doesn't support
733
# the compression_level parameter.
734
if self.loose_compression_level not in (-1, None):
735
raw_string = obj.as_legacy_object(
736
compression_level=self.loose_compression_level)
738
raw_string = obj.as_legacy_object()
739
self.transport.put_bytes(path, raw_string)
741
def move_in_pack(self, f):
742
"""Move a specific file containing a pack into the pack directory.
744
:note: The file should be on the same file system as the
747
:param path: Path to the pack file.
750
p = PackData("", f, len(f.getvalue()))
751
entries = p.sorted_entries()
752
basename = "pack-%s" % iter_sha1(entry[0]
753
for entry in entries).decode('ascii')
754
p._filename = basename + ".pack"
756
self.pack_transport.put_file(basename + ".pack", f)
757
with self.pack_transport.open_write_stream(basename + ".idx") as idxfile:
758
write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
759
idxfile = self.pack_transport.get(basename + ".idx")
760
idx = load_pack_index_file(basename + ".idx", idxfile)
761
final_pack = Pack.from_objects(p, idx)
762
final_pack._basename = basename
763
self._add_cached_pack(basename, final_pack)
766
def move_in_thin_pack(self, f):
767
"""Move a specific file containing a pack into the pack directory.
769
:note: The file should be on the same file system as the
772
:param path: Path to the pack file.
775
p = Pack('', resolve_ext_ref=self.get_raw)
776
p._data = PackData.from_file(f, len(f.getvalue()))
778
p._idx_load = lambda: MemoryPackIndex(
779
p.data.sorted_entries(), p.data.get_stored_checksum())
781
pack_sha = p.index.objects_sha1()
783
with self.pack_transport.open_write_stream(
784
"pack-%s.pack" % pack_sha.decode('ascii')) as datafile:
785
entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
786
entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
787
with self.pack_transport.open_write_stream(
788
"pack-%s.idx" % pack_sha.decode('ascii')) as idxfile:
789
write_pack_index_v2(idxfile, entries, data_sum)
792
"""Add a new pack to this object store.
794
:return: Fileobject to write to and a commit function to
795
call when the pack is finished.
800
if len(f.getvalue()) > 0:
801
return self.move_in_pack(f)
807
return f, commit, abort
810
def init(cls, transport):
812
transport.mkdir('info')
816
transport.mkdir(PACKDIR)
819
return cls(transport)
added
removed
breezy/git/transportgit.py
