/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/git/transportgit.py

  • Committer: Jelmer Vernooij
  • Date: 2018-08-14 01:15:02 UTC
  • mto: This revision was merged to the branch mainline in revision 7078.
  • Revision ID: jelmer@jelmer.uk-20180814011502-5zaydaq02vc2qxo1
Fix tests.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/git/transportgit.py
 
1
# Copyright (C) 2010-2018 Jelmer Vernooij <jelmer@jelmer.uk>
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
16
 
 
17
"""A Git repository implementation that uses a Bazaar transport."""
 
18
 
 
19
from __future__ import absolute_import
 
20
 
 
21
from io import BytesIO
 
22
 
 
23
import os
 
24
import sys
 
25
 
 
26
from dulwich.errors import (
 
27
    NotGitRepository,
 
28
    NoIndexPresent,
 
29
    )
 
30
from dulwich.file import (
 
31
    GitFile,
 
32
    FileLocked,
 
33
    )
 
34
from dulwich.objects import (
 
35
    ShaFile,
 
36
    )
 
37
from dulwich.object_store import (
 
38
    PackBasedObjectStore,
 
39
    PACKDIR,
 
40
    )
 
41
from dulwich.pack import (
 
42
    MemoryPackIndex,
 
43
    PackData,
 
44
    Pack,
 
45
    iter_sha1,
 
46
    load_pack_index_file,
 
47
    write_pack_objects,
 
48
    write_pack_index_v2,
 
49
    )
 
50
from dulwich.repo import (
 
51
    BaseRepo,
 
52
    InfoRefsContainer,
 
53
    RefsContainer,
 
54
    BASE_DIRECTORIES,
 
55
    COMMONDIR,
 
56
    CONTROLDIR,
 
57
    INDEX_FILENAME,
 
58
    OBJECTDIR,
 
59
    REFSDIR,
 
60
    SYMREF,
 
61
    check_ref_format,
 
62
    read_packed_refs_with_peeled,
 
63
    read_packed_refs,
 
64
    write_packed_refs,
 
65
    )
 
66
 
 
67
from .. import (
 
68
    osutils,
 
69
    transport as _mod_transport,
 
70
    urlutils,
 
71
    )
 
72
from ..sixish import (
 
73
    PY3,
 
74
    text_type,
 
75
    )
 
76
from ..errors import (
 
77
    AlreadyControlDirError,
 
78
    FileExists,
 
79
    LockBroken,
 
80
    LockError,
 
81
    LockContention,
 
82
    NotLocalUrl,
 
83
    NoSuchFile,
 
84
    ReadError,
 
85
    TransportNotPossible,
 
86
    )
 
87
 
 
88
from ..lock import LogicalLockResult
 
89
 
 
90
 
 
91
class TransportRefsContainer(RefsContainer):
 
92
    """Refs container that reads refs from a transport."""
 
93
 
 
94
    def __init__(self, transport, worktree_transport=None):
 
95
        self.transport = transport
 
96
        if worktree_transport is None:
 
97
            worktree_transport = transport
 
98
        self.worktree_transport = worktree_transport
 
99
        self._packed_refs = None
 
100
        self._peeled_refs = None
 
101
 
 
102
    def __repr__(self):
 
103
        return "%s(%r)" % (self.__class__.__name__, self.transport)
 
104
 
 
105
    def _ensure_dir_exists(self, path):
 
106
        for n in range(path.count("/")):
 
107
            dirname = "/".join(path.split("/")[:n+1])
 
108
            try:
 
109
                self.transport.mkdir(dirname)
 
110
            except FileExists:
 
111
                pass
 
112
 
 
113
    def subkeys(self, base):
 
114
        """Refs present in this container under a base.
 
115
 
 
116
        :param base: The base to return refs under.
 
117
        :return: A set of valid refs in this container under the base; the base
 
118
            prefix is stripped from the ref names returned.
 
119
        """
 
120
        keys = set()
 
121
        base_len = len(base) + 1
 
122
        for refname in self.allkeys():
 
123
            if refname.startswith(base):
 
124
                keys.add(refname[base_len:])
 
125
        return keys
 
126
 
 
127
    def allkeys(self):
 
128
        keys = set()
 
129
        try:
 
130
            self.worktree_transport.get_bytes("HEAD")
 
131
        except NoSuchFile:
 
132
            pass
 
133
        else:
 
134
            keys.add(b"HEAD")
 
135
        try:
 
136
            iter_files = list(self.transport.clone("refs").iter_files_recursive())
 
137
            for filename in iter_files:
 
138
                unquoted_filename = urlutils.unquote_to_bytes(filename)
 
139
                refname = osutils.pathjoin(b"refs", unquoted_filename)
 
140
                if check_ref_format(refname):
 
141
                    keys.add(refname)
 
142
        except (TransportNotPossible, NoSuchFile):
 
143
            pass
 
144
        keys.update(self.get_packed_refs())
 
145
        return keys
 
146
 
 
147
    def get_packed_refs(self):
 
148
        """Get contents of the packed-refs file.
 
149
 
 
150
        :return: Dictionary mapping ref names to SHA1s
 
151
 
 
152
        :note: Will return an empty dictionary when no packed-refs file is
 
153
            present.
 
154
        """
 
155
        # TODO: invalidate the cache on repacking
 
156
        if self._packed_refs is None:
 
157
            # set both to empty because we want _peeled_refs to be
 
158
            # None if and only if _packed_refs is also None.
 
159
            self._packed_refs = {}
 
160
            self._peeled_refs = {}
 
161
            try:
 
162
                f = self.transport.get("packed-refs")
 
163
            except NoSuchFile:
 
164
                return {}
 
165
            try:
 
166
                first_line = next(iter(f)).rstrip()
 
167
                if (first_line.startswith("# pack-refs") and " peeled" in
 
168
                        first_line):
 
169
                    for sha, name, peeled in read_packed_refs_with_peeled(f):
 
170
                        self._packed_refs[name] = sha
 
171
                        if peeled:
 
172
                            self._peeled_refs[name] = peeled
 
173
                else:
 
174
                    f.seek(0)
 
175
                    for sha, name in read_packed_refs(f):
 
176
                        self._packed_refs[name] = sha
 
177
            finally:
 
178
                f.close()
 
179
        return self._packed_refs
 
180
 
 
181
    def get_peeled(self, name):
 
182
        """Return the cached peeled value of a ref, if available.
 
183
 
 
184
        :param name: Name of the ref to peel
 
185
        :return: The peeled value of the ref. If the ref is known not point to a
 
186
            tag, this will be the SHA the ref refers to. If the ref may point to
 
187
            a tag, but no cached information is available, None is returned.
 
188
        """
 
189
        self.get_packed_refs()
 
190
        if self._peeled_refs is None or name not in self._packed_refs:
 
191
            # No cache: no peeled refs were read, or this ref is loose
 
192
            return None
 
193
        if name in self._peeled_refs:
 
194
            return self._peeled_refs[name]
 
195
        else:
 
196
            # Known not peelable
 
197
            return self[name]
 
198
 
 
199
    def read_loose_ref(self, name):
 
200
        """Read a reference file and return its contents.
 
201
 
 
202
        If the reference file a symbolic reference, only read the first line of
 
203
        the file. Otherwise, only read the first 40 bytes.
 
204
 
 
205
        :param name: the refname to read, relative to refpath
 
206
        :return: The contents of the ref file, or None if the file does not
 
207
            exist.
 
208
        :raises IOError: if any other error occurs
 
209
        """
 
210
        if name == b'HEAD':
 
211
            transport = self.worktree_transport
 
212
        else:
 
213
            transport = self.transport
 
214
        try:
 
215
            f = transport.get(urlutils.quote_from_bytes(name))
 
216
        except NoSuchFile:
 
217
            return None
 
218
        with f:
 
219
            header = f.read(len(SYMREF))
 
220
            if header == SYMREF:
 
221
                # Read only the first line
 
222
                return header + next(iter(f)).rstrip(b"\r\n")
 
223
            else:
 
224
                # Read only the first 40 bytes
 
225
                return header + f.read(40-len(SYMREF))
 
226
 
 
227
    def _remove_packed_ref(self, name):
 
228
        if self._packed_refs is None:
 
229
            return
 
230
        # reread cached refs from disk, while holding the lock
 
231
 
 
232
        self._packed_refs = None
 
233
        self.get_packed_refs()
 
234
 
 
235
        if name not in self._packed_refs:
 
236
            return
 
237
 
 
238
        del self._packed_refs[name]
 
239
        if name in self._peeled_refs:
 
240
            del self._peeled_refs[name]
 
241
        f = self.transport.open_write_stream("packed-refs")
 
242
        try:
 
243
            write_packed_refs(f, self._packed_refs, self._peeled_refs)
 
244
        finally:
 
245
            f.close()
 
246
 
 
247
    def set_symbolic_ref(self, name, other):
 
248
        """Make a ref point at another ref.
 
249
 
 
250
        :param name: Name of the ref to set
 
251
        :param other: Name of the ref to point at
 
252
        """
 
253
        self._check_refname(name)
 
254
        self._check_refname(other)
 
255
        if name != b'HEAD':
 
256
            transport = self.transport
 
257
            self._ensure_dir_exists(urlutils.quote_from_bytes(name))
 
258
        else:
 
259
            transport = self.worktree_transport
 
260
        transport.put_bytes(urlutils.quote_from_bytes(name), SYMREF + other + b'\n')
 
261
 
 
262
    def set_if_equals(self, name, old_ref, new_ref):
 
263
        """Set a refname to new_ref only if it currently equals old_ref.
 
264
 
 
265
        This method follows all symbolic references, and can be used to perform
 
266
        an atomic compare-and-swap operation.
 
267
 
 
268
        :param name: The refname to set.
 
269
        :param old_ref: The old sha the refname must refer to, or None to set
 
270
            unconditionally.
 
271
        :param new_ref: The new sha the refname will refer to.
 
272
        :return: True if the set was successful, False otherwise.
 
273
        """
 
274
        try:
 
275
            realnames, _ = self.follow(name)
 
276
            realname = realnames[-1]
 
277
        except (KeyError, IndexError):
 
278
            realname = name
 
279
        if realname == b'HEAD':
 
280
            transport = self.worktree_transport
 
281
        else:
 
282
            transport = self.transport
 
283
            self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
 
284
        transport.put_bytes(urlutils.quote_from_bytes(realname), new_ref+b"\n")
 
285
        return True
 
286
 
 
287
    def add_if_new(self, name, ref):
 
288
        """Add a new reference only if it does not already exist.
 
289
 
 
290
        This method follows symrefs, and only ensures that the last ref in the
 
291
        chain does not exist.
 
292
 
 
293
        :param name: The refname to set.
 
294
        :param ref: The new sha the refname will refer to.
 
295
        :return: True if the add was successful, False otherwise.
 
296
        """
 
297
        try:
 
298
            realnames, contents = self.follow(name)
 
299
            if contents is not None:
 
300
                return False
 
301
            realname = realnames[-1]
 
302
        except (KeyError, IndexError):
 
303
            realname = name
 
304
        self._check_refname(realname)
 
305
        if realname == b'HEAD':
 
306
            transport = self.worktree_transport
 
307
        else:
 
308
            transport = self.transport
 
309
            self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
 
310
        transport.put_bytes(urlutils.quote_from_bytes(realname), ref+b"\n")
 
311
        return True
 
312
 
 
313
    def remove_if_equals(self, name, old_ref):
 
314
        """Remove a refname only if it currently equals old_ref.
 
315
 
 
316
        This method does not follow symbolic references. It can be used to
 
317
        perform an atomic compare-and-delete operation.
 
318
 
 
319
        :param name: The refname to delete.
 
320
        :param old_ref: The old sha the refname must refer to, or None to delete
 
321
            unconditionally.
 
322
        :return: True if the delete was successful, False otherwise.
 
323
        """
 
324
        self._check_refname(name)
 
325
        # may only be packed
 
326
        if name == b'HEAD':
 
327
            transport = self.worktree_transport
 
328
        else:
 
329
            transport = self.transport
 
330
        try:
 
331
            transport.delete(urlutils.quote_from_bytes(name))
 
332
        except NoSuchFile:
 
333
            pass
 
334
        self._remove_packed_ref(name)
 
335
        return True
 
336
 
 
337
    def get(self, name, default=None):
 
338
        try:
 
339
            return self[name]
 
340
        except KeyError:
 
341
            return default
 
342
 
 
343
    def unlock_ref(self, name):
 
344
        if name == b"HEAD":
 
345
            transport = self.worktree_transport
 
346
        else:
 
347
            transport = self.transport
 
348
        lockname = name + b".lock"
 
349
        try:
 
350
            self.transport.delete(urlutils.quote_from_bytes(lockname))
 
351
        except NoSuchFile:
 
352
            pass
 
353
 
 
354
    def lock_ref(self, name):
 
355
        if name == b"HEAD":
 
356
            transport = self.worktree_transport
 
357
        else:
 
358
            transport = self.transport
 
359
        self._ensure_dir_exists(urlutils.quote_from_bytes(name))
 
360
        lockname = urlutils.quote_from_bytes(name + b".lock")
 
361
        try:
 
362
            local_path = self.transport.local_abspath(urlutils.quote_from_bytes(name))
 
363
        except NotLocalUrl:
 
364
            # This is racy, but what can we do?
 
365
            if self.transport.has(lockname):
 
366
                raise LockContention(name)
 
367
            lock_result = self.transport.put_bytes(lockname, b'Locked by brz-git')
 
368
            return LogicalLockResult(lambda: self.transport.delete(lockname))
 
369
        else:
 
370
            try:
 
371
                gf = GitFile(local_path, 'wb')
 
372
            except FileLocked as e:
 
373
                raise LockContention(name, e)
 
374
            else:
 
375
                def unlock():
 
376
                    try:
 
377
                        self.transport.delete(lockname)
 
378
                    except NoSuchFile:
 
379
                        raise LockBroken(lockname)
 
380
                    # GitFile.abort doesn't care if the lock has already disappeared
 
381
                    gf.abort()
 
382
                return LogicalLockResult(unlock)
 
383
 
 
384
 
 
385
# TODO(jelmer): Use upstream read_gitfile; unfortunately that expects strings
 
386
# rather than bytes..
 
387
def read_gitfile(f):
 
388
    """Read a ``.git`` file.
 
389
 
 
390
    The first line of the file should start with "gitdir: "
 
391
 
 
392
    :param f: File-like object to read from
 
393
    :return: A path
 
394
    """
 
395
    cs = f.read()
 
396
    if not cs.startswith(b"gitdir: "):
 
397
        raise ValueError("Expected file to start with 'gitdir: '")
 
398
    return cs[len(b"gitdir: "):].rstrip(b"\n")
 
399
 
 
400
 
 
401
class TransportRepo(BaseRepo):
 
402
 
 
403
    def __init__(self, transport, bare, refs_text=None):
 
404
        self.transport = transport
 
405
        self.bare = bare
 
406
        try:
 
407
            with transport.get(CONTROLDIR) as f:
 
408
                path = read_gitfile(f)
 
409
        except (ReadError, NoSuchFile):
 
410
            if self.bare:
 
411
                self._controltransport = self.transport
 
412
            else:
 
413
                self._controltransport = self.transport.clone('.git')
 
414
        else:
 
415
            self._controltransport = self.transport.clone(urlutils.quote_from_bytes(path))
 
416
        commondir = self.get_named_file(COMMONDIR)
 
417
        if commondir is not None:
 
418
            with commondir:
 
419
                commondir = os.path.join(
 
420
                    self.controldir(),
 
421
                    commondir.read().rstrip(b"\r\n").decode(
 
422
                        sys.getfilesystemencoding()))
 
423
                self._commontransport = \
 
424
                    _mod_transport.get_transport_from_path(commondir)
 
425
        else:
 
426
            self._commontransport = self._controltransport
 
427
        object_store = TransportObjectStore(
 
428
            self._commontransport.clone(OBJECTDIR))
 
429
        if refs_text is not None:
 
430
            refs_container = InfoRefsContainer(BytesIO(refs_text))
 
431
            try:
 
432
                head = TransportRefsContainer(self._commontransport).read_loose_ref("HEAD")
 
433
            except KeyError:
 
434
                pass
 
435
            else:
 
436
                refs_container._refs["HEAD"] = head
 
437
        else:
 
438
            refs_container = TransportRefsContainer(
 
439
                    self._commontransport, self._controltransport)
 
440
        super(TransportRepo, self).__init__(object_store,
 
441
                refs_container)
 
442
 
 
443
    def controldir(self):
 
444
        return self._controltransport.local_abspath('.')
 
445
 
 
446
    def commondir(self):
 
447
        return self._commontransport.local_abspath('.')
 
448
 
 
449
    @property
 
450
    def path(self):
 
451
        return self.transport.local_abspath('.')
 
452
 
 
453
    def _determine_file_mode(self):
 
454
        # Be consistent with bzr
 
455
        if sys.platform == 'win32':
 
456
            return False
 
457
        return True
 
458
 
 
459
    def get_named_file(self, path):
 
460
        """Get a file from the control dir with a specific name.
 
461
 
 
462
        Although the filename should be interpreted as a filename relative to
 
463
        the control dir in a disk-baked Repo, the object returned need not be
 
464
        pointing to a file in that location.
 
465
 
 
466
        :param path: The path to the file, relative to the control dir.
 
467
        :return: An open file object, or None if the file does not exist.
 
468
        """
 
469
        try:
 
470
            return self._controltransport.get(path.lstrip('/'))
 
471
        except NoSuchFile:
 
472
            return None
 
473
 
 
474
    def _put_named_file(self, relpath, contents):
 
475
        self._controltransport.put_bytes(relpath, contents)
 
476
 
 
477
    def index_path(self):
 
478
        """Return the path to the index file."""
 
479
        return self._controltransport.local_abspath(INDEX_FILENAME)
 
480
 
 
481
    def open_index(self):
 
482
        """Open the index for this repository."""
 
483
        from dulwich.index import Index
 
484
        if not self.has_index():
 
485
            raise NoIndexPresent()
 
486
        return Index(self.index_path())
 
487
 
 
488
    def has_index(self):
 
489
        """Check if an index is present."""
 
490
        # Bare repos must never have index files; non-bare repos may have a
 
491
        # missing index file, which is treated as empty.
 
492
        return not self.bare
 
493
 
 
494
    def get_config(self):
 
495
        from dulwich.config import ConfigFile
 
496
        try:
 
497
            with self._controltransport.get('config') as f:
 
498
                return ConfigFile.from_file(f)
 
499
        except NoSuchFile:
 
500
            return ConfigFile()
 
501
 
 
502
    def get_config_stack(self):
 
503
        from dulwich.config import StackedConfig
 
504
        backends = []
 
505
        p = self.get_config()
 
506
        if p is not None:
 
507
            backends.append(p)
 
508
            writable = p
 
509
        else:
 
510
            writable = None
 
511
        backends.extend(StackedConfig.default_backends())
 
512
        return StackedConfig(backends, writable=writable)
 
513
 
 
514
    def __repr__(self):
 
515
        return "<%s for %r>" % (self.__class__.__name__, self.transport)
 
516
 
 
517
    @classmethod
 
518
    def init(cls, transport, bare=False):
 
519
        if not bare:
 
520
            try:
 
521
                transport.mkdir(".git")
 
522
            except FileExists:
 
523
                raise AlreadyControlDirError(transport.base)
 
524
            control_transport = transport.clone(".git")
 
525
        else:
 
526
            control_transport = transport
 
527
        for d in BASE_DIRECTORIES:
 
528
            try:
 
529
                control_transport.mkdir("/".join(d))
 
530
            except FileExists:
 
531
                pass
 
532
        try:
 
533
            control_transport.mkdir(OBJECTDIR)
 
534
        except FileExists:
 
535
            raise AlreadyControlDirError(transport.base)
 
536
        TransportObjectStore.init(control_transport.clone(OBJECTDIR))
 
537
        ret = cls(transport, bare)
 
538
        ret.refs.set_symbolic_ref(b"HEAD", b"refs/heads/master")
 
539
        ret._init_files(bare)
 
540
        return ret
 
541
 
 
542
 
 
543
class TransportObjectStore(PackBasedObjectStore):
 
544
    """Git-style object store that exists on disk."""
 
545
 
 
546
    def __init__(self, transport):
 
547
        """Open an object store.
 
548
 
 
549
        :param transport: Transport to open data from
 
550
        """
 
551
        super(TransportObjectStore, self).__init__()
 
552
        self.transport = transport
 
553
        self.pack_transport = self.transport.clone(PACKDIR)
 
554
        self._alternates = None
 
555
 
 
556
    def __eq__(self, other):
 
557
        if not isinstance(other, TransportObjectStore):
 
558
            return False
 
559
        return self.transport == other.transport
 
560
 
 
561
    def __repr__(self):
 
562
        return "%s(%r)" % (self.__class__.__name__, self.transport)
 
563
 
 
564
    @property
 
565
    def alternates(self):
 
566
        if self._alternates is not None:
 
567
            return self._alternates
 
568
        self._alternates = []
 
569
        for path in self._read_alternate_paths():
 
570
            # FIXME: Check path
 
571
            t = _mod_transport.get_transport_from_path(path)
 
572
            self._alternates.append(self.__class__(t))
 
573
        return self._alternates
 
574
 
 
575
    def _read_alternate_paths(self):
 
576
        try:
 
577
            f = self.transport.get("info/alternates")
 
578
        except NoSuchFile:
 
579
            return []
 
580
        ret = []
 
581
        with f:
 
582
            for l in f.read().splitlines():
 
583
                if l[0] == b"#":
 
584
                    continue
 
585
                if os.path.isabs(l):
 
586
                    continue
 
587
                ret.append(l)
 
588
            return ret
 
589
 
 
590
    @property
 
591
    def packs(self):
 
592
        # FIXME: Never invalidates.
 
593
        if not self._pack_cache:
 
594
            self._update_pack_cache()
 
595
        return self._pack_cache.values()
 
596
 
 
597
    def _update_pack_cache(self):
 
598
        for pack in self._load_packs():
 
599
            self._pack_cache[pack._basename] = pack
 
600
 
 
601
    def _pack_names(self):
 
602
        try:
 
603
            f = self.transport.get('info/packs')
 
604
        except NoSuchFile:
 
605
            return self.pack_transport.list_dir(".")
 
606
        else:
 
607
            with f:
 
608
                ret = []
 
609
                for line in f.read().splitlines():
 
610
                    if not line:
 
611
                        continue
 
612
                    (kind, name) = line.split(b" ", 1)
 
613
                    if kind != b"P":
 
614
                        continue
 
615
                    ret.append(name)
 
616
                return ret
 
617
 
 
618
    def _remove_pack(self, pack):
 
619
        self.pack_transport.delete(os.path.basename(pack.index.path))
 
620
        self.pack_transport.delete(pack.data.filename)
 
621
 
 
622
    def _load_packs(self):
 
623
        ret = []
 
624
        for name in self._pack_names():
 
625
            if name.startswith("pack-") and name.endswith(".pack"):
 
626
                try:
 
627
                    size = self.pack_transport.stat(name).st_size
 
628
                except TransportNotPossible:
 
629
                    f = self.pack_transport.get(name)
 
630
                    pd = PackData(name, f, size=len(contents))
 
631
                else:
 
632
                    pd = PackData(name, self.pack_transport.get(name),
 
633
                            size=size)
 
634
                idxname = name.replace(".pack", ".idx")
 
635
                idx = load_pack_index_file(idxname, self.pack_transport.get(idxname))
 
636
                pack = Pack.from_objects(pd, idx)
 
637
                pack._basename = idxname[:-4]
 
638
                ret.append(pack)
 
639
        return ret
 
640
 
 
641
    def _iter_loose_objects(self):
 
642
        for base in self.transport.list_dir('.'):
 
643
            if len(base) != 2:
 
644
                continue
 
645
            for rest in self.transport.list_dir(base):
 
646
                yield (base+rest).encode(sys.getfilesystemencoding())
 
647
 
 
648
    def _split_loose_object(self, sha):
 
649
        return (sha[:2], sha[2:])
 
650
 
 
651
    def _remove_loose_object(self, sha):
 
652
        path = osutils.joinpath(self._split_loose_object(sha))
 
653
        self.transport.delete(urlutils.quote_from_bytes(path))
 
654
 
 
655
    def _get_loose_object(self, sha):
 
656
        path = osutils.joinpath(self._split_loose_object(sha))
 
657
        try:
 
658
            with self.transport.get(urlutils.quote_from_bytes(path)) as f:
 
659
                return ShaFile.from_file(f)
 
660
        except NoSuchFile:
 
661
            return None
 
662
 
 
663
    def add_object(self, obj):
 
664
        """Add a single object to this object store.
 
665
 
 
666
        :param obj: Object to add
 
667
        """
 
668
        (dir, file) = self._split_loose_object(obj.id)
 
669
        try:
 
670
            self.transport.mkdir(urlutils.quote_from_bytes(dir))
 
671
        except FileExists:
 
672
            pass
 
673
        path = urlutils.quote_from_bytes(osutils.pathjoin(dir, file))
 
674
        if self.transport.has(path):
 
675
            return # Already there, no need to write again
 
676
        self.transport.put_bytes(path, obj.as_legacy_object())
 
677
 
 
678
    def move_in_pack(self, f):
 
679
        """Move a specific file containing a pack into the pack directory.
 
680
 
 
681
        :note: The file should be on the same file system as the
 
682
            packs directory.
 
683
 
 
684
        :param path: Path to the pack file.
 
685
        """
 
686
        f.seek(0)
 
687
        p = PackData("", f, len(f.getvalue()))
 
688
        entries = p.sorted_entries()
 
689
        basename = "pack-%s" % iter_sha1(entry[0] for entry in entries).decode('ascii')
 
690
        p._filename = basename + ".pack"
 
691
        f.seek(0)
 
692
        self.pack_transport.put_file(basename + ".pack", f)
 
693
        idxfile = self.pack_transport.open_write_stream(basename + ".idx")
 
694
        try:
 
695
            write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
 
696
        finally:
 
697
            idxfile.close()
 
698
        idxfile = self.pack_transport.get(basename + ".idx")
 
699
        idx = load_pack_index_file(basename+".idx", idxfile)
 
700
        final_pack = Pack.from_objects(p, idx)
 
701
        final_pack._basename = basename
 
702
        self._add_known_pack(basename, final_pack)
 
703
        return final_pack
 
704
 
 
705
    def move_in_thin_pack(self, f):
 
706
        """Move a specific file containing a pack into the pack directory.
 
707
 
 
708
        :note: The file should be on the same file system as the
 
709
            packs directory.
 
710
 
 
711
        :param path: Path to the pack file.
 
712
        """
 
713
        f.seek(0)
 
714
        p = Pack('', resolve_ext_ref=self.get_raw)
 
715
        p._data = PackData.from_file(f, len(f.getvalue()))
 
716
        p._data.pack = p
 
717
        p._idx_load = lambda: MemoryPackIndex(p.data.sorted_entries(), p.data.get_stored_checksum())
 
718
 
 
719
        pack_sha = p.index.objects_sha1()
 
720
 
 
721
        datafile = self.pack_transport.open_write_stream(
 
722
                "pack-%s.pack" % pack_sha.decode('ascii'))
 
723
        try:
 
724
            entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
 
725
        finally:
 
726
            datafile.close()
 
727
        entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
 
728
        idxfile = self.pack_transport.open_write_stream(
 
729
            "pack-%s.idx" % pack_sha.decode('ascii'))
 
730
        try:
 
731
            write_pack_index_v2(idxfile, entries, data_sum)
 
732
        finally:
 
733
            idxfile.close()
 
734
        # TODO(jelmer): Just add new pack to the cache
 
735
        self._flush_pack_cache()
 
736
 
 
737
    def add_pack(self):
 
738
        """Add a new pack to this object store.
 
739
 
 
740
        :return: Fileobject to write to and a commit function to
 
741
            call when the pack is finished.
 
742
        """
 
743
        f = BytesIO()
 
744
        def commit():
 
745
            if len(f.getvalue()) > 0:
 
746
                return self.move_in_pack(f)
 
747
            else:
 
748
                return None
 
749
        def abort():
 
750
            return None
 
751
        return f, commit, abort
 
752
 
 
753
    @classmethod
 
754
    def init(cls, transport):
 
755
        try:
 
756
            transport.mkdir('info')
 
757
        except FileExists:
 
758
            pass
 
759
        try:
 
760
            transport.mkdir(PACKDIR)
 
761
        except FileExists:
 
762
            pass
 
763
        return cls(transport)