1
# Copyright (C) 2010-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""A Git repository implementation that uses a Bazaar transport."""
19
from __future__ import absolute_import
21
from io import BytesIO
26
from dulwich.errors import (
29
from dulwich.file import (
33
from dulwich.objects import (
36
from dulwich.object_store import (
40
from dulwich.pack import (
49
from dulwich.repo import (
60
read_packed_refs_with_peeled,
67
transport as _mod_transport,
70
from ..errors import (
71
AlreadyControlDirError,
81
from ..lock import LogicalLockResult
84
class TransportRefsContainer(RefsContainer):
85
"""Refs container that reads refs from a transport."""
87
def __init__(self, transport, worktree_transport=None):
88
self.transport = transport
89
if worktree_transport is None:
90
worktree_transport = transport
91
self.worktree_transport = worktree_transport
92
self._packed_refs = None
93
self._peeled_refs = None
96
return "%s(%r)" % (self.__class__.__name__, self.transport)
98
def _ensure_dir_exists(self, path):
99
for n in range(path.count("/")):
100
dirname = "/".join(path.split("/")[:n + 1])
102
self.transport.mkdir(dirname)
106
def subkeys(self, base):
107
"""Refs present in this container under a base.
109
:param base: The base to return refs under.
110
:return: A set of valid refs in this container under the base; the base
111
prefix is stripped from the ref names returned.
114
base_len = len(base) + 1
115
for refname in self.allkeys():
116
if refname.startswith(base):
117
keys.add(refname[base_len:])
123
self.worktree_transport.get_bytes("HEAD")
129
iter_files = list(self.transport.clone(
130
"refs").iter_files_recursive())
131
for filename in iter_files:
132
unquoted_filename = urlutils.unquote_to_bytes(filename)
133
refname = osutils.pathjoin(b"refs", unquoted_filename)
134
if check_ref_format(refname):
136
except (TransportNotPossible, NoSuchFile):
138
keys.update(self.get_packed_refs())
141
def get_packed_refs(self):
142
"""Get contents of the packed-refs file.
144
:return: Dictionary mapping ref names to SHA1s
146
:note: Will return an empty dictionary when no packed-refs file is
149
# TODO: invalidate the cache on repacking
150
if self._packed_refs is None:
151
# set both to empty because we want _peeled_refs to be
152
# None if and only if _packed_refs is also None.
153
self._packed_refs = {}
154
self._peeled_refs = {}
156
f = self.transport.get("packed-refs")
160
first_line = next(iter(f)).rstrip()
161
if (first_line.startswith(b"# pack-refs") and b" peeled" in
163
for sha, name, peeled in read_packed_refs_with_peeled(f):
164
self._packed_refs[name] = sha
166
self._peeled_refs[name] = peeled
169
for sha, name in read_packed_refs(f):
170
self._packed_refs[name] = sha
173
return self._packed_refs
175
def get_peeled(self, name):
176
"""Return the cached peeled value of a ref, if available.
178
:param name: Name of the ref to peel
179
:return: The peeled value of the ref. If the ref is known not point to
180
a tag, this will be the SHA the ref refers to. If the ref may point
181
to a tag, but no cached information is available, None is returned.
183
self.get_packed_refs()
184
if self._peeled_refs is None or name not in self._packed_refs:
185
# No cache: no peeled refs were read, or this ref is loose
187
if name in self._peeled_refs:
188
return self._peeled_refs[name]
193
def read_loose_ref(self, name):
194
"""Read a reference file and return its contents.
196
If the reference file a symbolic reference, only read the first line of
197
the file. Otherwise, only read the first 40 bytes.
199
:param name: the refname to read, relative to refpath
200
:return: The contents of the ref file, or None if the file does not
202
:raises IOError: if any other error occurs
205
transport = self.worktree_transport
207
transport = self.transport
209
f = transport.get(urlutils.quote_from_bytes(name))
213
header = f.read(len(SYMREF))
215
# Read only the first line
216
return header + next(iter(f)).rstrip(b"\r\n")
218
# Read only the first 40 bytes
219
return header + f.read(40 - len(SYMREF))
221
def _remove_packed_ref(self, name):
222
if self._packed_refs is None:
224
# reread cached refs from disk, while holding the lock
226
self._packed_refs = None
227
self.get_packed_refs()
229
if name not in self._packed_refs:
232
del self._packed_refs[name]
233
if name in self._peeled_refs:
234
del self._peeled_refs[name]
235
f = self.transport.open_write_stream("packed-refs")
237
write_packed_refs(f, self._packed_refs, self._peeled_refs)
241
def set_symbolic_ref(self, name, other):
242
"""Make a ref point at another ref.
244
:param name: Name of the ref to set
245
:param other: Name of the ref to point at
247
self._check_refname(name)
248
self._check_refname(other)
250
transport = self.transport
251
self._ensure_dir_exists(urlutils.quote_from_bytes(name))
253
transport = self.worktree_transport
254
transport.put_bytes(urlutils.quote_from_bytes(
255
name), SYMREF + other + b'\n')
257
def set_if_equals(self, name, old_ref, new_ref):
258
"""Set a refname to new_ref only if it currently equals old_ref.
260
This method follows all symbolic references, and can be used to perform
261
an atomic compare-and-swap operation.
263
:param name: The refname to set.
264
:param old_ref: The old sha the refname must refer to, or None to set
266
:param new_ref: The new sha the refname will refer to.
267
:return: True if the set was successful, False otherwise.
270
realnames, _ = self.follow(name)
271
realname = realnames[-1]
272
except (KeyError, IndexError):
274
if realname == b'HEAD':
275
transport = self.worktree_transport
277
transport = self.transport
278
self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
279
transport.put_bytes(urlutils.quote_from_bytes(
280
realname), new_ref + b"\n")
283
def add_if_new(self, name, ref):
284
"""Add a new reference only if it does not already exist.
286
This method follows symrefs, and only ensures that the last ref in the
287
chain does not exist.
289
:param name: The refname to set.
290
:param ref: The new sha the refname will refer to.
291
:return: True if the add was successful, False otherwise.
294
realnames, contents = self.follow(name)
295
if contents is not None:
297
realname = realnames[-1]
298
except (KeyError, IndexError):
300
self._check_refname(realname)
301
if realname == b'HEAD':
302
transport = self.worktree_transport
304
transport = self.transport
305
self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
306
transport.put_bytes(urlutils.quote_from_bytes(realname), ref + b"\n")
309
def remove_if_equals(self, name, old_ref):
310
"""Remove a refname only if it currently equals old_ref.
312
This method does not follow symbolic references. It can be used to
313
perform an atomic compare-and-delete operation.
315
:param name: The refname to delete.
316
:param old_ref: The old sha the refname must refer to, or None to
317
delete unconditionally.
318
:return: True if the delete was successful, False otherwise.
320
self._check_refname(name)
323
transport = self.worktree_transport
325
transport = self.transport
327
transport.delete(urlutils.quote_from_bytes(name))
330
self._remove_packed_ref(name)
333
def get(self, name, default=None):
339
def unlock_ref(self, name):
341
transport = self.worktree_transport
343
transport = self.transport
344
lockname = name + b".lock"
346
transport.delete(urlutils.quote_from_bytes(lockname))
350
def lock_ref(self, name):
352
transport = self.worktree_transport
354
transport = self.transport
355
self._ensure_dir_exists(urlutils.quote_from_bytes(name))
356
lockname = urlutils.quote_from_bytes(name + b".lock")
358
local_path = transport.local_abspath(
359
urlutils.quote_from_bytes(name))
361
# This is racy, but what can we do?
362
if transport.has(lockname):
363
raise LockContention(name)
364
transport.put_bytes(lockname, b'Locked by brz-git')
365
return LogicalLockResult(lambda: transport.delete(lockname))
368
gf = GitFile(local_path, 'wb')
369
except FileLocked as e:
370
raise LockContention(name, e)
374
transport.delete(lockname)
376
raise LockBroken(lockname)
377
# GitFile.abort doesn't care if the lock has already
380
return LogicalLockResult(unlock)
383
# TODO(jelmer): Use upstream read_gitfile; unfortunately that expects strings
384
# rather than bytes..
386
"""Read a ``.git`` file.
388
The first line of the file should start with "gitdir: "
390
:param f: File-like object to read from
394
if not cs.startswith(b"gitdir: "):
395
raise ValueError("Expected file to start with 'gitdir: '")
396
return cs[len(b"gitdir: "):].rstrip(b"\n")
399
class TransportRepo(BaseRepo):
401
def __init__(self, transport, bare, refs_text=None):
402
self.transport = transport
405
with transport.get(CONTROLDIR) as f:
406
path = read_gitfile(f)
407
except (ReadError, NoSuchFile):
409
self._controltransport = self.transport
411
self._controltransport = self.transport.clone('.git')
413
self._controltransport = self.transport.clone(
414
urlutils.quote_from_bytes(path))
415
commondir = self.get_named_file(COMMONDIR)
416
if commondir is not None:
418
commondir = os.path.join(
420
commondir.read().rstrip(b"\r\n").decode(
421
sys.getfilesystemencoding()))
422
self._commontransport = \
423
_mod_transport.get_transport_from_path(commondir)
425
self._commontransport = self._controltransport
426
object_store = TransportObjectStore(
427
self._commontransport.clone(OBJECTDIR))
428
if refs_text is not None:
429
refs_container = InfoRefsContainer(BytesIO(refs_text))
431
head = TransportRefsContainer(
432
self._commontransport).read_loose_ref("HEAD")
436
refs_container._refs["HEAD"] = head
438
refs_container = TransportRefsContainer(
439
self._commontransport, self._controltransport)
440
super(TransportRepo, self).__init__(object_store,
443
def controldir(self):
444
return self._controltransport.local_abspath('.')
447
return self._commontransport.local_abspath('.')
451
return self.transport.local_abspath('.')
453
def _determine_file_mode(self):
454
# Be consistent with bzr
455
if sys.platform == 'win32':
459
def get_named_file(self, path):
460
"""Get a file from the control dir with a specific name.
462
Although the filename should be interpreted as a filename relative to
463
the control dir in a disk-baked Repo, the object returned need not be
464
pointing to a file in that location.
466
:param path: The path to the file, relative to the control dir.
467
:return: An open file object, or None if the file does not exist.
470
return self._controltransport.get(path.lstrip('/'))
474
def _put_named_file(self, relpath, contents):
475
self._controltransport.put_bytes(relpath, contents)
477
def index_path(self):
478
"""Return the path to the index file."""
479
return self._controltransport.local_abspath(INDEX_FILENAME)
481
def open_index(self):
482
"""Open the index for this repository."""
483
from dulwich.index import Index
484
if not self.has_index():
485
raise NoIndexPresent()
486
return Index(self.index_path())
489
"""Check if an index is present."""
490
# Bare repos must never have index files; non-bare repos may have a
491
# missing index file, which is treated as empty.
494
def get_config(self):
495
from dulwich.config import ConfigFile
497
with self._controltransport.get('config') as f:
498
return ConfigFile.from_file(f)
502
def get_config_stack(self):
503
from dulwich.config import StackedConfig
505
p = self.get_config()
511
backends.extend(StackedConfig.default_backends())
512
return StackedConfig(backends, writable=writable)
515
return "<%s for %r>" % (self.__class__.__name__, self.transport)
518
def init(cls, transport, bare=False):
521
transport.mkdir(".git")
523
raise AlreadyControlDirError(transport.base)
524
control_transport = transport.clone(".git")
526
control_transport = transport
527
for d in BASE_DIRECTORIES:
529
control_transport.mkdir("/".join(d))
533
control_transport.mkdir(OBJECTDIR)
535
raise AlreadyControlDirError(transport.base)
536
TransportObjectStore.init(control_transport.clone(OBJECTDIR))
537
ret = cls(transport, bare)
538
ret.refs.set_symbolic_ref(b"HEAD", b"refs/heads/master")
539
ret._init_files(bare)
543
class TransportObjectStore(PackBasedObjectStore):
544
"""Git-style object store that exists on disk."""
546
def __init__(self, transport):
547
"""Open an object store.
549
:param transport: Transport to open data from
551
super(TransportObjectStore, self).__init__()
552
self.transport = transport
553
self.pack_transport = self.transport.clone(PACKDIR)
554
self._alternates = None
556
def __eq__(self, other):
557
if not isinstance(other, TransportObjectStore):
559
return self.transport == other.transport
562
return "%s(%r)" % (self.__class__.__name__, self.transport)
565
def alternates(self):
566
if self._alternates is not None:
567
return self._alternates
568
self._alternates = []
569
for path in self._read_alternate_paths():
571
t = _mod_transport.get_transport_from_path(path)
572
self._alternates.append(self.__class__(t))
573
return self._alternates
575
def _read_alternate_paths(self):
577
f = self.transport.get("info/alternates")
582
for l in f.read().splitlines():
592
# FIXME: Never invalidates.
593
if not self._pack_cache:
594
self._update_pack_cache()
595
return self._pack_cache.values()
597
def _update_pack_cache(self):
598
for pack in self._load_packs():
599
self._pack_cache[pack._basename] = pack
601
def _pack_names(self):
603
return self.pack_transport.list_dir(".")
604
except TransportNotPossible:
606
f = self.transport.get('info/packs')
608
# Hmm, warn about running 'git update-server-info' ?
611
# TODO(jelmer): Move to top-level after dulwich
612
# 0.19.7 is released.
613
from dulwich.object_store import read_packs_file
615
return read_packs_file(f)
619
def _remove_pack(self, pack):
620
self.pack_transport.delete(os.path.basename(pack.index.path))
621
self.pack_transport.delete(pack.data.filename)
623
def _load_packs(self):
625
for name in self._pack_names():
626
if name.startswith("pack-") and name.endswith(".pack"):
628
size = self.pack_transport.stat(name).st_size
629
except TransportNotPossible:
630
f = self.pack_transport.get(name)
631
pd = PackData(name, f)
633
pd = PackData(name, self.pack_transport.get(name),
635
idxname = name.replace(".pack", ".idx")
636
idx = load_pack_index_file(
637
idxname, self.pack_transport.get(idxname))
638
pack = Pack.from_objects(pd, idx)
639
pack._basename = idxname[:-4]
643
def _iter_loose_objects(self):
644
for base in self.transport.list_dir('.'):
647
for rest in self.transport.list_dir(base):
648
yield (base + rest).encode(sys.getfilesystemencoding())
650
def _split_loose_object(self, sha):
651
return (sha[:2], sha[2:])
653
def _remove_loose_object(self, sha):
654
path = osutils.joinpath(self._split_loose_object(sha))
655
self.transport.delete(urlutils.quote_from_bytes(path))
657
def _get_loose_object(self, sha):
658
path = osutils.joinpath(self._split_loose_object(sha))
660
with self.transport.get(urlutils.quote_from_bytes(path)) as f:
661
return ShaFile.from_file(f)
665
def add_object(self, obj):
666
"""Add a single object to this object store.
668
:param obj: Object to add
670
(dir, file) = self._split_loose_object(obj.id)
672
self.transport.mkdir(urlutils.quote_from_bytes(dir))
675
path = urlutils.quote_from_bytes(osutils.pathjoin(dir, file))
676
if self.transport.has(path):
677
return # Already there, no need to write again
678
self.transport.put_bytes(path, obj.as_legacy_object())
680
def move_in_pack(self, f):
681
"""Move a specific file containing a pack into the pack directory.
683
:note: The file should be on the same file system as the
686
:param path: Path to the pack file.
689
p = PackData("", f, len(f.getvalue()))
690
entries = p.sorted_entries()
691
basename = "pack-%s" % iter_sha1(entry[0]
692
for entry in entries).decode('ascii')
693
p._filename = basename + ".pack"
695
self.pack_transport.put_file(basename + ".pack", f)
696
idxfile = self.pack_transport.open_write_stream(basename + ".idx")
698
write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
701
idxfile = self.pack_transport.get(basename + ".idx")
702
idx = load_pack_index_file(basename + ".idx", idxfile)
703
final_pack = Pack.from_objects(p, idx)
704
final_pack._basename = basename
705
self._add_known_pack(basename, final_pack)
708
def move_in_thin_pack(self, f):
709
"""Move a specific file containing a pack into the pack directory.
711
:note: The file should be on the same file system as the
714
:param path: Path to the pack file.
717
p = Pack('', resolve_ext_ref=self.get_raw)
718
p._data = PackData.from_file(f, len(f.getvalue()))
720
p._idx_load = lambda: MemoryPackIndex(
721
p.data.sorted_entries(), p.data.get_stored_checksum())
723
pack_sha = p.index.objects_sha1()
725
datafile = self.pack_transport.open_write_stream(
726
"pack-%s.pack" % pack_sha.decode('ascii'))
728
entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
731
entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
732
idxfile = self.pack_transport.open_write_stream(
733
"pack-%s.idx" % pack_sha.decode('ascii'))
735
write_pack_index_v2(idxfile, entries, data_sum)
738
# TODO(jelmer): Just add new pack to the cache
739
self._flush_pack_cache()
742
"""Add a new pack to this object store.
744
:return: Fileobject to write to and a commit function to
745
call when the pack is finished.
750
if len(f.getvalue()) > 0:
751
return self.move_in_pack(f)
757
return f, commit, abort
760
def init(cls, transport):
762
transport.mkdir('info')
766
transport.mkdir(PACKDIR)
769
return cls(transport)