1
# Copyright (C) 2010 Jelmer Vernooij <jelmer@samba.org>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""A Git repository implementation that uses a Bazaar transport."""
19
from cStringIO import StringIO
21
from dulwich.errors import (
25
from dulwich.objects import (
28
from dulwich.object_store import (
32
from dulwich.pack import (
41
from dulwich.repo import (
50
read_packed_refs_with_peeled,
55
from bzrlib.errors import (
62
class TransportRefsContainer(RefsContainer):
63
"""Refs container that reads refs from a transport."""
65
def __init__(self, transport):
66
self.transport = transport
67
self._packed_refs = None
68
self._peeled_refs = None
71
return "%s(%r)" % (self.__class__.__name__, self.transport)
73
def _ensure_dir_exists(self, path):
74
for n in range(path.count("/")):
75
dirname = "/".join(path.split("/")[:n+1])
77
self.transport.mkdir(dirname)
81
def subkeys(self, base):
84
iter_files = self.transport.clone(base).iter_files_recursive()
85
keys.update(("%s/%s" % (base, refname)).strip("/") for
86
refname in iter_files if check_ref_format("%s/%s" % (base, refname)))
87
except (TransportNotPossible, NoSuchFile):
89
for key in self.get_packed_refs():
90
if key.startswith(base):
91
keys.add(key[len(base):].strip("/"))
96
if self.transport.has("HEAD"):
99
iter_files = list(self.transport.clone("refs").iter_files_recursive())
100
for filename in iter_files:
101
refname = "refs/%s" % filename
102
if check_ref_format(refname):
104
except (TransportNotPossible, NoSuchFile):
106
keys.update(self.get_packed_refs())
109
def get_packed_refs(self):
110
"""Get contents of the packed-refs file.
112
:return: Dictionary mapping ref names to SHA1s
114
:note: Will return an empty dictionary when no packed-refs file is
117
# TODO: invalidate the cache on repacking
118
if self._packed_refs is None:
119
# set both to empty because we want _peeled_refs to be
120
# None if and only if _packed_refs is also None.
121
self._packed_refs = {}
122
self._peeled_refs = {}
124
f = self.transport.get("packed-refs")
128
first_line = iter(f).next().rstrip()
129
if (first_line.startswith("# pack-refs") and " peeled" in
131
for sha, name, peeled in read_packed_refs_with_peeled(f):
132
self._packed_refs[name] = sha
134
self._peeled_refs[name] = peeled
137
for sha, name in read_packed_refs(f):
138
self._packed_refs[name] = sha
141
return self._packed_refs
143
def get_peeled(self, name):
144
"""Return the cached peeled value of a ref, if available.
146
:param name: Name of the ref to peel
147
:return: The peeled value of the ref. If the ref is known not point to a
148
tag, this will be the SHA the ref refers to. If the ref may point to
149
a tag, but no cached information is available, None is returned.
151
self.get_packed_refs()
152
if self._peeled_refs is None or name not in self._packed_refs:
153
# No cache: no peeled refs were read, or this ref is loose
155
if name in self._peeled_refs:
156
return self._peeled_refs[name]
161
def read_loose_ref(self, name):
162
"""Read a reference file and return its contents.
164
If the reference file a symbolic reference, only read the first line of
165
the file. Otherwise, only read the first 40 bytes.
167
:param name: the refname to read, relative to refpath
168
:return: The contents of the ref file, or None if the file does not
170
:raises IOError: if any other error occurs
173
f = self.transport.get(name)
177
header = f.read(len(SYMREF))
179
# Read only the first line
180
return header + iter(f).next().rstrip("\r\n")
182
# Read only the first 40 bytes
183
return header + f.read(40-len(SYMREF))
187
def _remove_packed_ref(self, name):
188
if self._packed_refs is None:
190
# reread cached refs from disk, while holding the lock
192
self._packed_refs = None
193
self.get_packed_refs()
195
if name not in self._packed_refs:
198
del self._packed_refs[name]
199
if name in self._peeled_refs:
200
del self._peeled_refs[name]
201
f = self.transport.open_write_stream("packed-refs")
203
write_packed_refs(f, self._packed_refs, self._peeled_refs)
207
def set_symbolic_ref(self, name, other):
208
"""Make a ref point at another ref.
210
:param name: Name of the ref to set
211
:param other: Name of the ref to point at
213
self._check_refname(name)
214
self._check_refname(other)
215
self._ensure_dir_exists(name)
216
self.transport.put_bytes(name, SYMREF + other + '\n')
218
def set_if_equals(self, name, old_ref, new_ref):
219
"""Set a refname to new_ref only if it currently equals old_ref.
221
This method follows all symbolic references, and can be used to perform
222
an atomic compare-and-swap operation.
224
:param name: The refname to set.
225
:param old_ref: The old sha the refname must refer to, or None to set
227
:param new_ref: The new sha the refname will refer to.
228
:return: True if the set was successful, False otherwise.
231
realname, _ = self._follow(name)
234
self._ensure_dir_exists(realname)
235
self.transport.put_bytes(realname, new_ref+"\n")
238
def add_if_new(self, name, ref):
239
"""Add a new reference only if it does not already exist.
241
This method follows symrefs, and only ensures that the last ref in the
242
chain does not exist.
244
:param name: The refname to set.
245
:param ref: The new sha the refname will refer to.
246
:return: True if the add was successful, False otherwise.
249
realname, contents = self._follow(name)
250
if contents is not None:
254
self._check_refname(realname)
255
self._ensure_dir_exists(realname)
256
self.transport.put_bytes(realname, ref+"\n")
259
def remove_if_equals(self, name, old_ref):
260
"""Remove a refname only if it currently equals old_ref.
262
This method does not follow symbolic references. It can be used to
263
perform an atomic compare-and-delete operation.
265
:param name: The refname to delete.
266
:param old_ref: The old sha the refname must refer to, or None to delete
268
:return: True if the delete was successful, False otherwise.
270
self._check_refname(name)
273
self.transport.delete(name)
276
self._remove_packed_ref(name)
280
class TransportRepo(BaseRepo):
282
def __init__(self, transport):
283
self.transport = transport
285
if self.transport.has(".git/%s" % OBJECTDIR):
287
self._controltransport = self.transport.clone('.git')
288
elif self.transport.has_any(["info/refs", OBJECTDIR, REFSDIR]):
290
self._controltransport = self.transport
292
raise NotGitRepository(self.transport)
294
raise NotGitRepository(self.transport)
295
object_store = TransportObjectStore(
296
self._controltransport.clone(OBJECTDIR))
297
super(TransportRepo, self).__init__(object_store,
298
TransportRefsContainer(self._controltransport))
300
def get_named_file(self, path):
301
"""Get a file from the control dir with a specific name.
303
Although the filename should be interpreted as a filename relative to
304
the control dir in a disk-baked Repo, the object returned need not be
305
pointing to a file in that location.
307
:param path: The path to the file, relative to the control dir.
308
:return: An open file object, or None if the file does not exist.
311
return self._controltransport.get(path.lstrip('/'))
315
def _put_named_file(self, relpath, contents):
316
self._controltransport.put_bytes(relpath, contents)
318
def index_path(self):
319
"""Return the path to the index file."""
320
return self._controltransport.local_abspath(INDEX_FILENAME)
322
def open_index(self):
323
"""Open the index for this repository."""
324
from dulwich.index import Index
325
if not self.has_index():
326
raise NoIndexPresent()
327
return Index(self.index_path())
330
"""Check if an index is present."""
331
# Bare repos must never have index files; non-bare repos may have a
332
# missing index file, which is treated as empty.
336
return "<%s for %r>" % (self.__class__.__name__, self.transport)
339
def init(cls, transport, bare=False):
341
transport.mkdir(".git")
342
control_transport = transport.clone(".git")
344
control_transport = transport
345
for d in BASE_DIRECTORIES:
346
control_transport.mkdir("/".join(d))
347
control_transport.mkdir(OBJECTDIR)
348
TransportObjectStore.init(control_transport.clone(OBJECTDIR))
350
ret.refs.set_symbolic_ref("HEAD", "refs/heads/master")
351
ret._init_files(bare)
355
class TransportObjectStore(PackBasedObjectStore):
356
"""Git-style object store that exists on disk."""
358
def __init__(self, transport):
359
"""Open an object store.
361
:param transport: Transport to open data from
363
super(TransportObjectStore, self).__init__()
364
self.transport = transport
365
self.pack_transport = self.transport.clone(PACKDIR)
368
return "%s(%r)" % (self.__class__.__name__, self.transport)
370
def _pack_cache_stale(self):
373
def _pack_names(self):
375
f = self.transport.get('info/packs')
377
return self.pack_transport.list_dir(".")
380
for line in f.readlines():
381
line = line.rstrip("\n")
384
(kind, name) = line.split(" ", 1)
390
def _load_packs(self):
392
for name in self._pack_names():
393
if name.startswith("pack-") and name.endswith(".pack"):
395
size = self.pack_transport.stat(name).st_size
396
except TransportNotPossible:
397
# FIXME: This reads the whole pack file at once
398
f = self.pack_transport.get(name)
400
pd = PackData(name, StringIO(contents), size=len(contents))
402
pd = PackData(name, self.pack_transport.get(name),
404
idxname = name.replace(".pack", ".idx")
405
idx = load_pack_index_file(idxname, self.pack_transport.get(idxname))
406
pack = Pack.from_objects(pd, idx)
410
def _iter_loose_objects(self):
411
for base in self.transport.list_dir('.'):
414
for rest in self.transport.list_dir(base):
417
def _split_loose_object(self, sha):
418
return (sha[:2], sha[2:])
420
def _remove_loose_object(self, sha):
421
path = '%s/%s' % self._split_loose_object(sha)
422
self.transport.delete(path)
424
def _get_loose_object(self, sha):
425
path = '%s/%s' % self._split_loose_object(sha)
427
return ShaFile.from_file(self.transport.get(path))
431
def add_object(self, obj):
432
"""Add a single object to this object store.
434
:param obj: Object to add
436
(dir, file) = self._split_loose_object(obj.id)
438
self.transport.mkdir(dir)
441
path = "%s/%s" % (dir, file)
442
if self.transport.has(path):
443
return # Already there, no need to write again
444
self.transport.put_bytes(path, obj.as_legacy_object())
446
def move_in_pack(self, f):
447
"""Move a specific file containing a pack into the pack directory.
449
:note: The file should be on the same file system as the
452
:param path: Path to the pack file.
455
p = PackData(None, f, len(f.getvalue()))
456
entries = p.sorted_entries()
457
basename = "pack-%s" % iter_sha1(entry[0] for entry in entries)
459
self.pack_transport.put_file(basename + ".pack", f)
460
idxfile = self.pack_transport.open_write_stream(basename + ".idx")
462
write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
465
idxfile = self.pack_transport.get(basename + ".idx")
466
idx = load_pack_index_file(basename+".idx", idxfile)
467
final_pack = Pack.from_objects(p, idx)
468
self._add_known_pack(final_pack)
471
def add_thin_pack(self):
472
"""Add a new thin pack to this object store.
474
Thin packs are packs that contain deltas with parents that exist
477
from cStringIO import StringIO
480
if len(f.getvalue()) > 0:
481
return self.move_in_thin_pack(f)
486
def move_in_thin_pack(self, f):
487
"""Move a specific file containing a pack into the pack directory.
489
:note: The file should be on the same file system as the
492
:param path: Path to the pack file.
495
data = PackData.from_file(self.get_raw, f, len(f.getvalue()))
496
idx = MemoryPackIndex(data.sorted_entries(), data.get_stored_checksum())
497
p = Pack.from_objects(data, idx)
499
pack_sha = idx.objects_sha1()
501
datafile = self.pack_transport.open_write_stream(
502
"pack-%s.pack" % pack_sha)
504
entries, data_sum = write_pack_data(datafile, p.pack_tuples())
508
idxfile = self.pack_transport.open_write_stream(
509
"pack-%s.idx" % pack_sha)
511
write_pack_index_v2(idxfile, data.sorted_entries(), data_sum)
514
final_pack = Pack("pack-%s" % pack_sha)
515
self._add_known_pack(final_pack)
521
"""Add a new pack to this object store.
523
:return: Fileobject to write to and a commit function to
524
call when the pack is finished.
526
from cStringIO import StringIO
529
if len(f.getvalue()) > 0:
530
return self.move_in_pack(f)
536
def init(cls, transport):
537
transport.mkdir('info')
538
transport.mkdir(PACKDIR)
539
return cls(transport)
added
removed
transportgit.py
