1
# Copyright (C) 2010-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""A Git repository implementation that uses a Bazaar transport."""
19
from io import BytesIO
24
from dulwich.errors import (
27
from dulwich.file import (
31
from dulwich.objects import (
34
from dulwich.object_store import (
39
from dulwich.pack import (
48
from dulwich.repo import (
59
read_packed_refs_with_peeled,
66
transport as _mod_transport,
69
from ..errors import (
70
AlreadyControlDirError,
80
from ..lock import LogicalLockResult
81
from ..trace import warning
84
class TransportRefsContainer(RefsContainer):
85
"""Refs container that reads refs from a transport."""
87
def __init__(self, transport, worktree_transport=None):
88
self.transport = transport
89
if worktree_transport is None:
90
worktree_transport = transport
91
self.worktree_transport = worktree_transport
92
self._packed_refs = None
93
self._peeled_refs = None
96
return "%s(%r)" % (self.__class__.__name__, self.transport)
98
def _ensure_dir_exists(self, path):
99
for n in range(path.count("/")):
100
dirname = "/".join(path.split("/")[:n + 1])
102
self.transport.mkdir(dirname)
106
def subkeys(self, base):
107
"""Refs present in this container under a base.
109
:param base: The base to return refs under.
110
:return: A set of valid refs in this container under the base; the base
111
prefix is stripped from the ref names returned.
114
base_len = len(base) + 1
115
for refname in self.allkeys():
116
if refname.startswith(base):
117
keys.add(refname[base_len:])
123
self.worktree_transport.get_bytes("HEAD")
129
iter_files = list(self.transport.clone(
130
"refs").iter_files_recursive())
131
for filename in iter_files:
132
unquoted_filename = urlutils.unquote_to_bytes(filename)
133
refname = osutils.pathjoin(b"refs", unquoted_filename)
134
if check_ref_format(refname):
136
except (TransportNotPossible, NoSuchFile):
138
keys.update(self.get_packed_refs())
141
def get_packed_refs(self):
142
"""Get contents of the packed-refs file.
144
:return: Dictionary mapping ref names to SHA1s
146
:note: Will return an empty dictionary when no packed-refs file is
149
# TODO: invalidate the cache on repacking
150
if self._packed_refs is None:
151
# set both to empty because we want _peeled_refs to be
152
# None if and only if _packed_refs is also None.
153
self._packed_refs = {}
154
self._peeled_refs = {}
156
f = self.transport.get("packed-refs")
160
first_line = next(iter(f)).rstrip()
161
if (first_line.startswith(b"# pack-refs") and b" peeled" in
163
for sha, name, peeled in read_packed_refs_with_peeled(f):
164
self._packed_refs[name] = sha
166
self._peeled_refs[name] = peeled
169
for sha, name in read_packed_refs(f):
170
self._packed_refs[name] = sha
173
return self._packed_refs
175
def get_peeled(self, name):
176
"""Return the cached peeled value of a ref, if available.
178
:param name: Name of the ref to peel
179
:return: The peeled value of the ref. If the ref is known not point to
180
a tag, this will be the SHA the ref refers to. If the ref may point
181
to a tag, but no cached information is available, None is returned.
183
self.get_packed_refs()
184
if self._peeled_refs is None or name not in self._packed_refs:
185
# No cache: no peeled refs were read, or this ref is loose
187
if name in self._peeled_refs:
188
return self._peeled_refs[name]
193
def read_loose_ref(self, name):
194
"""Read a reference file and return its contents.
196
If the reference file a symbolic reference, only read the first line of
197
the file. Otherwise, only read the first 40 bytes.
199
:param name: the refname to read, relative to refpath
200
:return: The contents of the ref file, or None if the file does not
202
:raises IOError: if any other error occurs
205
transport = self.worktree_transport
207
transport = self.transport
209
f = transport.get(urlutils.quote_from_bytes(name))
214
header = f.read(len(SYMREF))
216
# probably a directory
219
# Read only the first line
220
return header + next(iter(f)).rstrip(b"\r\n")
222
# Read only the first 40 bytes
223
return header + f.read(40 - len(SYMREF))
225
def _remove_packed_ref(self, name):
226
if self._packed_refs is None:
228
# reread cached refs from disk, while holding the lock
230
self._packed_refs = None
231
self.get_packed_refs()
233
if name not in self._packed_refs:
236
del self._packed_refs[name]
237
if name in self._peeled_refs:
238
del self._peeled_refs[name]
239
with self.transport.open_write_stream("packed-refs") as f:
240
write_packed_refs(f, self._packed_refs, self._peeled_refs)
242
def set_symbolic_ref(self, name, other):
243
"""Make a ref point at another ref.
245
:param name: Name of the ref to set
246
:param other: Name of the ref to point at
248
self._check_refname(name)
249
self._check_refname(other)
251
transport = self.transport
252
self._ensure_dir_exists(urlutils.quote_from_bytes(name))
254
transport = self.worktree_transport
255
transport.put_bytes(urlutils.quote_from_bytes(
256
name), SYMREF + other + b'\n')
258
def set_if_equals(self, name, old_ref, new_ref):
259
"""Set a refname to new_ref only if it currently equals old_ref.
261
This method follows all symbolic references, and can be used to perform
262
an atomic compare-and-swap operation.
264
:param name: The refname to set.
265
:param old_ref: The old sha the refname must refer to, or None to set
267
:param new_ref: The new sha the refname will refer to.
268
:return: True if the set was successful, False otherwise.
271
realnames, _ = self.follow(name)
272
realname = realnames[-1]
273
except (KeyError, IndexError):
275
if realname == b'HEAD':
276
transport = self.worktree_transport
278
transport = self.transport
279
self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
280
transport.put_bytes(urlutils.quote_from_bytes(
281
realname), new_ref + b"\n")
284
def add_if_new(self, name, ref):
285
"""Add a new reference only if it does not already exist.
287
This method follows symrefs, and only ensures that the last ref in the
288
chain does not exist.
290
:param name: The refname to set.
291
:param ref: The new sha the refname will refer to.
292
:return: True if the add was successful, False otherwise.
295
realnames, contents = self.follow(name)
296
if contents is not None:
298
realname = realnames[-1]
299
except (KeyError, IndexError):
301
self._check_refname(realname)
302
if realname == b'HEAD':
303
transport = self.worktree_transport
305
transport = self.transport
306
self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
307
transport.put_bytes(urlutils.quote_from_bytes(realname), ref + b"\n")
310
def remove_if_equals(self, name, old_ref):
311
"""Remove a refname only if it currently equals old_ref.
313
This method does not follow symbolic references. It can be used to
314
perform an atomic compare-and-delete operation.
316
:param name: The refname to delete.
317
:param old_ref: The old sha the refname must refer to, or None to
318
delete unconditionally.
319
:return: True if the delete was successful, False otherwise.
321
self._check_refname(name)
324
transport = self.worktree_transport
326
transport = self.transport
328
transport.delete(urlutils.quote_from_bytes(name))
331
self._remove_packed_ref(name)
334
def get(self, name, default=None):
340
def unlock_ref(self, name):
342
transport = self.worktree_transport
344
transport = self.transport
345
lockname = name + b".lock"
347
transport.delete(urlutils.quote_from_bytes(lockname))
351
def lock_ref(self, name):
353
transport = self.worktree_transport
355
transport = self.transport
356
self._ensure_dir_exists(urlutils.quote_from_bytes(name))
357
lockname = urlutils.quote_from_bytes(name + b".lock")
359
local_path = transport.local_abspath(
360
urlutils.quote_from_bytes(name))
362
# This is racy, but what can we do?
363
if transport.has(lockname):
364
raise LockContention(name)
365
transport.put_bytes(lockname, b'Locked by brz-git')
366
return LogicalLockResult(lambda: transport.delete(lockname))
369
gf = GitFile(local_path, 'wb')
370
except FileLocked as e:
371
raise LockContention(name, e)
375
transport.delete(lockname)
377
raise LockBroken(lockname)
378
# GitFile.abort doesn't care if the lock has already
381
return LogicalLockResult(unlock)
384
# TODO(jelmer): Use upstream read_gitfile; unfortunately that expects strings
385
# rather than bytes..
387
"""Read a ``.git`` file.
389
The first line of the file should start with "gitdir: "
391
:param f: File-like object to read from
395
if not cs.startswith(b"gitdir: "):
396
raise ValueError("Expected file to start with 'gitdir: '")
397
return cs[len(b"gitdir: "):].rstrip(b"\n")
400
class TransportRepo(BaseRepo):
402
def __init__(self, transport, bare, refs_text=None):
403
self.transport = transport
406
with transport.get(CONTROLDIR) as f:
407
path = read_gitfile(f)
408
except (ReadError, NoSuchFile):
410
self._controltransport = self.transport
412
self._controltransport = self.transport.clone('.git')
414
self._controltransport = self.transport.clone(
415
urlutils.quote_from_bytes(path))
416
commondir = self.get_named_file(COMMONDIR)
417
if commondir is not None:
419
commondir = os.path.join(
421
commondir.read().rstrip(b"\r\n").decode(
422
sys.getfilesystemencoding()))
423
self._commontransport = \
424
_mod_transport.get_transport_from_path(commondir)
426
self._commontransport = self._controltransport
427
object_store = TransportObjectStore(
428
self._commontransport.clone(OBJECTDIR))
429
if refs_text is not None:
430
refs_container = InfoRefsContainer(BytesIO(refs_text))
432
head = TransportRefsContainer(
433
self._commontransport).read_loose_ref(b"HEAD")
437
refs_container._refs[b"HEAD"] = head
439
refs_container = TransportRefsContainer(
440
self._commontransport, self._controltransport)
441
super(TransportRepo, self).__init__(object_store,
444
def controldir(self):
445
return self._controltransport.local_abspath('.')
448
return self._commontransport.local_abspath('.')
452
return self.transport.local_abspath('.')
454
def _determine_file_mode(self):
455
# Be consistent with bzr
456
if sys.platform == 'win32':
460
def get_named_file(self, path):
461
"""Get a file from the control dir with a specific name.
463
Although the filename should be interpreted as a filename relative to
464
the control dir in a disk-baked Repo, the object returned need not be
465
pointing to a file in that location.
467
:param path: The path to the file, relative to the control dir.
468
:return: An open file object, or None if the file does not exist.
471
return self._controltransport.get(path.lstrip('/'))
475
def _put_named_file(self, relpath, contents):
476
self._controltransport.put_bytes(relpath, contents)
478
def index_path(self):
479
"""Return the path to the index file."""
480
return self._controltransport.local_abspath(INDEX_FILENAME)
482
def open_index(self):
483
"""Open the index for this repository."""
484
from dulwich.index import Index
485
if not self.has_index():
486
raise NoIndexPresent()
487
return Index(self.index_path())
490
"""Check if an index is present."""
491
# Bare repos must never have index files; non-bare repos may have a
492
# missing index file, which is treated as empty.
495
def get_config(self):
496
from dulwich.config import ConfigFile
498
with self._controltransport.get('config') as f:
499
return ConfigFile.from_file(f)
503
def get_config_stack(self):
504
from dulwich.config import StackedConfig
506
p = self.get_config()
512
backends.extend(StackedConfig.default_backends())
513
return StackedConfig(backends, writable=writable)
516
return "<%s for %r>" % (self.__class__.__name__, self.transport)
519
def init(cls, transport, bare=False):
522
transport.mkdir(".git")
524
raise AlreadyControlDirError(transport.base)
525
control_transport = transport.clone(".git")
527
control_transport = transport
528
for d in BASE_DIRECTORIES:
530
control_transport.mkdir("/".join(d))
534
control_transport.mkdir(OBJECTDIR)
536
raise AlreadyControlDirError(transport.base)
537
TransportObjectStore.init(control_transport.clone(OBJECTDIR))
538
ret = cls(transport, bare)
539
ret.refs.set_symbolic_ref(b"HEAD", b"refs/heads/master")
540
ret._init_files(bare)
544
class TransportObjectStore(PackBasedObjectStore):
545
"""Git-style object store that exists on disk."""
547
def __init__(self, transport):
548
"""Open an object store.
550
:param transport: Transport to open data from
552
super(TransportObjectStore, self).__init__()
553
self.transport = transport
554
self.pack_transport = self.transport.clone(PACKDIR)
555
self._alternates = None
557
def __eq__(self, other):
558
if not isinstance(other, TransportObjectStore):
560
return self.transport == other.transport
563
return "%s(%r)" % (self.__class__.__name__, self.transport)
566
def alternates(self):
567
if self._alternates is not None:
568
return self._alternates
569
self._alternates = []
570
for path in self._read_alternate_paths():
572
t = _mod_transport.get_transport_from_path(path)
573
self._alternates.append(self.__class__(t))
574
return self._alternates
576
def _read_alternate_paths(self):
578
f = self.transport.get("info/alternates")
583
for l in f.read().splitlines():
591
def _update_pack_cache(self):
592
pack_files = set(self._pack_names())
594
for basename in pack_files:
595
pack_name = basename + ".pack"
596
if basename not in self._pack_cache:
598
size = self.pack_transport.stat(pack_name).st_size
599
except TransportNotPossible:
600
f = self.pack_transport.get(pack_name)
601
# TODO(jelmer): Don't read entire file into memory?
602
f = BytesIO(f.read())
603
pd = PackData(pack_name, f)
606
pack_name, self.pack_transport.get(pack_name),
608
idxname = basename + ".idx"
609
idx = load_pack_index_file(
610
idxname, self.pack_transport.get(idxname))
611
pack = Pack.from_objects(pd, idx)
612
pack._basename = basename
613
self._pack_cache[basename] = pack
614
new_packs.append(pack)
615
# Remove disappeared pack files
616
for f in set(self._pack_cache) - pack_files:
617
self._pack_cache.pop(f).close()
620
def _pack_names(self):
623
dir_contents = self.pack_transport.list_dir(".")
624
for name in dir_contents:
625
if name.startswith("pack-") and name.endswith(".pack"):
626
# verify that idx exists first (otherwise the pack was not yet
628
idx_name = os.path.splitext(name)[0] + ".idx"
629
if idx_name in dir_contents:
630
pack_files.append(os.path.splitext(name)[0])
631
except TransportNotPossible:
633
f = self.transport.get('info/packs')
635
warning('No info/packs on remote host;'
636
'run \'git update-server-info\' on remote.')
640
os.path.splitext(name)[0]
641
for name in read_packs_file(f)]
646
def _remove_pack(self, pack):
647
self.pack_transport.delete(os.path.basename(pack.index.path))
648
self.pack_transport.delete(pack.data.filename)
650
del self._pack_cache[os.path.basename(pack._basename)]
654
def _iter_loose_objects(self):
655
for base in self.transport.list_dir('.'):
658
for rest in self.transport.list_dir(base):
659
yield (base + rest).encode(sys.getfilesystemencoding())
661
def _split_loose_object(self, sha):
662
return (sha[:2], sha[2:])
664
def _remove_loose_object(self, sha):
665
path = osutils.joinpath(self._split_loose_object(sha))
666
self.transport.delete(urlutils.quote_from_bytes(path))
668
def _get_loose_object(self, sha):
669
path = osutils.joinpath(self._split_loose_object(sha))
671
with self.transport.get(urlutils.quote_from_bytes(path)) as f:
672
return ShaFile.from_file(f)
676
def add_object(self, obj):
677
"""Add a single object to this object store.
679
:param obj: Object to add
681
(dir, file) = self._split_loose_object(obj.id)
683
self.transport.mkdir(urlutils.quote_from_bytes(dir))
686
path = urlutils.quote_from_bytes(osutils.pathjoin(dir, file))
687
if self.transport.has(path):
688
return # Already there, no need to write again
689
self.transport.put_bytes(path, obj.as_legacy_object())
691
def move_in_pack(self, f):
692
"""Move a specific file containing a pack into the pack directory.
694
:note: The file should be on the same file system as the
697
:param path: Path to the pack file.
700
p = PackData("", f, len(f.getvalue()))
701
entries = p.sorted_entries()
702
basename = "pack-%s" % iter_sha1(entry[0]
703
for entry in entries).decode('ascii')
704
p._filename = basename + ".pack"
706
self.pack_transport.put_file(basename + ".pack", f)
707
with self.pack_transport.open_write_stream(basename + ".idx") as idxfile:
708
write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
709
idxfile = self.pack_transport.get(basename + ".idx")
710
idx = load_pack_index_file(basename + ".idx", idxfile)
711
final_pack = Pack.from_objects(p, idx)
712
final_pack._basename = basename
713
self._add_cached_pack(basename, final_pack)
716
def move_in_thin_pack(self, f):
717
"""Move a specific file containing a pack into the pack directory.
719
:note: The file should be on the same file system as the
722
:param path: Path to the pack file.
725
p = Pack('', resolve_ext_ref=self.get_raw)
726
p._data = PackData.from_file(f, len(f.getvalue()))
728
p._idx_load = lambda: MemoryPackIndex(
729
p.data.sorted_entries(), p.data.get_stored_checksum())
731
pack_sha = p.index.objects_sha1()
733
with self.pack_transport.open_write_stream(
734
"pack-%s.pack" % pack_sha.decode('ascii')) as datafile:
735
entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
736
entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
737
with self.pack_transport.open_write_stream(
738
"pack-%s.idx" % pack_sha.decode('ascii')) as idxfile:
739
write_pack_index_v2(idxfile, entries, data_sum)
742
"""Add a new pack to this object store.
744
:return: Fileobject to write to and a commit function to
745
call when the pack is finished.
750
if len(f.getvalue()) > 0:
751
return self.move_in_pack(f)
757
return f, commit, abort
760
def init(cls, transport):
762
transport.mkdir('info')
766
transport.mkdir(PACKDIR)
769
return cls(transport)
added
removed
breezy/git/transportgit.py
