/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/git/transportgit.py

  • Committer: Jelmer Vernooij
  • Date: 2020-05-05 23:32:39 UTC
  • mto: (7490.7.21 work)
  • mto: This revision was merged to the branch mainline in revision 7501.
  • Revision ID: jelmer@jelmer.uk-20200505233239-kdmnmscn8eisltk6
Add a breezy.__main__ module.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/git/transportgit.py
 
1
# Copyright (C) 2010-2018 Jelmer Vernooij <jelmer@jelmer.uk>
 
2
#
 
3
# This program is free software; you can redistribute it and/or modify
 
4
# it under the terms of the GNU General Public License as published by
 
5
# the Free Software Foundation; either version 2 of the License, or
 
6
# (at your option) any later version.
 
7
#
 
8
# This program is distributed in the hope that it will be useful,
 
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
11
# GNU General Public License for more details.
 
12
#
 
13
# You should have received a copy of the GNU General Public License
 
14
# along with this program; if not, write to the Free Software
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
16
 
 
17
"""A Git repository implementation that uses a Bazaar transport."""
 
18
 
 
19
from __future__ import absolute_import
 
20
 
 
21
from io import BytesIO
 
22
 
 
23
import os
 
24
import sys
 
25
 
 
26
from dulwich.errors import (
 
27
    NoIndexPresent,
 
28
    )
 
29
from dulwich.file import (
 
30
    GitFile,
 
31
    FileLocked,
 
32
    )
 
33
from dulwich.objects import (
 
34
    ShaFile,
 
35
    )
 
36
from dulwich.object_store import (
 
37
    PackBasedObjectStore,
 
38
    PACKDIR,
 
39
    read_packs_file,
 
40
    )
 
41
from dulwich.pack import (
 
42
    MemoryPackIndex,
 
43
    PackData,
 
44
    Pack,
 
45
    iter_sha1,
 
46
    load_pack_index_file,
 
47
    write_pack_objects,
 
48
    write_pack_index_v2,
 
49
    )
 
50
from dulwich.repo import (
 
51
    BaseRepo,
 
52
    InfoRefsContainer,
 
53
    RefsContainer,
 
54
    BASE_DIRECTORIES,
 
55
    COMMONDIR,
 
56
    CONTROLDIR,
 
57
    INDEX_FILENAME,
 
58
    OBJECTDIR,
 
59
    SYMREF,
 
60
    check_ref_format,
 
61
    read_packed_refs_with_peeled,
 
62
    read_packed_refs,
 
63
    write_packed_refs,
 
64
    )
 
65
 
 
66
from .. import (
 
67
    osutils,
 
68
    transport as _mod_transport,
 
69
    urlutils,
 
70
    )
 
71
from ..errors import (
 
72
    AlreadyControlDirError,
 
73
    FileExists,
 
74
    LockBroken,
 
75
    LockContention,
 
76
    NotLocalUrl,
 
77
    NoSuchFile,
 
78
    ReadError,
 
79
    TransportNotPossible,
 
80
    )
 
81
 
 
82
from ..lock import LogicalLockResult
 
83
from ..trace import warning
 
84
 
 
85
 
 
86
class TransportRefsContainer(RefsContainer):
 
87
    """Refs container that reads refs from a transport."""
 
88
 
 
89
    def __init__(self, transport, worktree_transport=None):
 
90
        self.transport = transport
 
91
        if worktree_transport is None:
 
92
            worktree_transport = transport
 
93
        self.worktree_transport = worktree_transport
 
94
        self._packed_refs = None
 
95
        self._peeled_refs = None
 
96
 
 
97
    def __repr__(self):
 
98
        return "%s(%r)" % (self.__class__.__name__, self.transport)
 
99
 
 
100
    def _ensure_dir_exists(self, path):
 
101
        for n in range(path.count("/")):
 
102
            dirname = "/".join(path.split("/")[:n + 1])
 
103
            try:
 
104
                self.transport.mkdir(dirname)
 
105
            except FileExists:
 
106
                pass
 
107
 
 
108
    def subkeys(self, base):
 
109
        """Refs present in this container under a base.
 
110
 
 
111
        :param base: The base to return refs under.
 
112
        :return: A set of valid refs in this container under the base; the base
 
113
            prefix is stripped from the ref names returned.
 
114
        """
 
115
        keys = set()
 
116
        base_len = len(base) + 1
 
117
        for refname in self.allkeys():
 
118
            if refname.startswith(base):
 
119
                keys.add(refname[base_len:])
 
120
        return keys
 
121
 
 
122
    def allkeys(self):
 
123
        keys = set()
 
124
        try:
 
125
            self.worktree_transport.get_bytes("HEAD")
 
126
        except NoSuchFile:
 
127
            pass
 
128
        else:
 
129
            keys.add(b"HEAD")
 
130
        try:
 
131
            iter_files = list(self.transport.clone(
 
132
                "refs").iter_files_recursive())
 
133
            for filename in iter_files:
 
134
                unquoted_filename = urlutils.unquote_to_bytes(filename)
 
135
                refname = osutils.pathjoin(b"refs", unquoted_filename)
 
136
                if check_ref_format(refname):
 
137
                    keys.add(refname)
 
138
        except (TransportNotPossible, NoSuchFile):
 
139
            pass
 
140
        keys.update(self.get_packed_refs())
 
141
        return keys
 
142
 
 
143
    def get_packed_refs(self):
 
144
        """Get contents of the packed-refs file.
 
145
 
 
146
        :return: Dictionary mapping ref names to SHA1s
 
147
 
 
148
        :note: Will return an empty dictionary when no packed-refs file is
 
149
            present.
 
150
        """
 
151
        # TODO: invalidate the cache on repacking
 
152
        if self._packed_refs is None:
 
153
            # set both to empty because we want _peeled_refs to be
 
154
            # None if and only if _packed_refs is also None.
 
155
            self._packed_refs = {}
 
156
            self._peeled_refs = {}
 
157
            try:
 
158
                f = self.transport.get("packed-refs")
 
159
            except NoSuchFile:
 
160
                return {}
 
161
            try:
 
162
                first_line = next(iter(f)).rstrip()
 
163
                if (first_line.startswith(b"# pack-refs") and b" peeled" in
 
164
                        first_line):
 
165
                    for sha, name, peeled in read_packed_refs_with_peeled(f):
 
166
                        self._packed_refs[name] = sha
 
167
                        if peeled:
 
168
                            self._peeled_refs[name] = peeled
 
169
                else:
 
170
                    f.seek(0)
 
171
                    for sha, name in read_packed_refs(f):
 
172
                        self._packed_refs[name] = sha
 
173
            finally:
 
174
                f.close()
 
175
        return self._packed_refs
 
176
 
 
177
    def get_peeled(self, name):
 
178
        """Return the cached peeled value of a ref, if available.
 
179
 
 
180
        :param name: Name of the ref to peel
 
181
        :return: The peeled value of the ref. If the ref is known not point to
 
182
            a tag, this will be the SHA the ref refers to. If the ref may point
 
183
            to a tag, but no cached information is available, None is returned.
 
184
        """
 
185
        self.get_packed_refs()
 
186
        if self._peeled_refs is None or name not in self._packed_refs:
 
187
            # No cache: no peeled refs were read, or this ref is loose
 
188
            return None
 
189
        if name in self._peeled_refs:
 
190
            return self._peeled_refs[name]
 
191
        else:
 
192
            # Known not peelable
 
193
            return self[name]
 
194
 
 
195
    def read_loose_ref(self, name):
 
196
        """Read a reference file and return its contents.
 
197
 
 
198
        If the reference file a symbolic reference, only read the first line of
 
199
        the file. Otherwise, only read the first 40 bytes.
 
200
 
 
201
        :param name: the refname to read, relative to refpath
 
202
        :return: The contents of the ref file, or None if the file does not
 
203
            exist.
 
204
        :raises IOError: if any other error occurs
 
205
        """
 
206
        if name == b'HEAD':
 
207
            transport = self.worktree_transport
 
208
        else:
 
209
            transport = self.transport
 
210
        try:
 
211
            f = transport.get(urlutils.quote_from_bytes(name))
 
212
        except NoSuchFile:
 
213
            return None
 
214
        with f:
 
215
            try:
 
216
                header = f.read(len(SYMREF))
 
217
            except ReadError:
 
218
                # probably a directory
 
219
                return None
 
220
            if header == SYMREF:
 
221
                # Read only the first line
 
222
                return header + next(iter(f)).rstrip(b"\r\n")
 
223
            else:
 
224
                # Read only the first 40 bytes
 
225
                return header + f.read(40 - len(SYMREF))
 
226
 
 
227
    def _remove_packed_ref(self, name):
 
228
        if self._packed_refs is None:
 
229
            return
 
230
        # reread cached refs from disk, while holding the lock
 
231
 
 
232
        self._packed_refs = None
 
233
        self.get_packed_refs()
 
234
 
 
235
        if name not in self._packed_refs:
 
236
            return
 
237
 
 
238
        del self._packed_refs[name]
 
239
        if name in self._peeled_refs:
 
240
            del self._peeled_refs[name]
 
241
        with self.transport.open_write_stream("packed-refs") as f:
 
242
            write_packed_refs(f, self._packed_refs, self._peeled_refs)
 
243
 
 
244
    def set_symbolic_ref(self, name, other):
 
245
        """Make a ref point at another ref.
 
246
 
 
247
        :param name: Name of the ref to set
 
248
        :param other: Name of the ref to point at
 
249
        """
 
250
        self._check_refname(name)
 
251
        self._check_refname(other)
 
252
        if name != b'HEAD':
 
253
            transport = self.transport
 
254
            self._ensure_dir_exists(urlutils.quote_from_bytes(name))
 
255
        else:
 
256
            transport = self.worktree_transport
 
257
        transport.put_bytes(urlutils.quote_from_bytes(
 
258
            name), SYMREF + other + b'\n')
 
259
 
 
260
    def set_if_equals(self, name, old_ref, new_ref):
 
261
        """Set a refname to new_ref only if it currently equals old_ref.
 
262
 
 
263
        This method follows all symbolic references, and can be used to perform
 
264
        an atomic compare-and-swap operation.
 
265
 
 
266
        :param name: The refname to set.
 
267
        :param old_ref: The old sha the refname must refer to, or None to set
 
268
            unconditionally.
 
269
        :param new_ref: The new sha the refname will refer to.
 
270
        :return: True if the set was successful, False otherwise.
 
271
        """
 
272
        try:
 
273
            realnames, _ = self.follow(name)
 
274
            realname = realnames[-1]
 
275
        except (KeyError, IndexError):
 
276
            realname = name
 
277
        if realname == b'HEAD':
 
278
            transport = self.worktree_transport
 
279
        else:
 
280
            transport = self.transport
 
281
            self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
 
282
        transport.put_bytes(urlutils.quote_from_bytes(
 
283
            realname), new_ref + b"\n")
 
284
        return True
 
285
 
 
286
    def add_if_new(self, name, ref):
 
287
        """Add a new reference only if it does not already exist.
 
288
 
 
289
        This method follows symrefs, and only ensures that the last ref in the
 
290
        chain does not exist.
 
291
 
 
292
        :param name: The refname to set.
 
293
        :param ref: The new sha the refname will refer to.
 
294
        :return: True if the add was successful, False otherwise.
 
295
        """
 
296
        try:
 
297
            realnames, contents = self.follow(name)
 
298
            if contents is not None:
 
299
                return False
 
300
            realname = realnames[-1]
 
301
        except (KeyError, IndexError):
 
302
            realname = name
 
303
        self._check_refname(realname)
 
304
        if realname == b'HEAD':
 
305
            transport = self.worktree_transport
 
306
        else:
 
307
            transport = self.transport
 
308
            self._ensure_dir_exists(urlutils.quote_from_bytes(realname))
 
309
        transport.put_bytes(urlutils.quote_from_bytes(realname), ref + b"\n")
 
310
        return True
 
311
 
 
312
    def remove_if_equals(self, name, old_ref):
 
313
        """Remove a refname only if it currently equals old_ref.
 
314
 
 
315
        This method does not follow symbolic references. It can be used to
 
316
        perform an atomic compare-and-delete operation.
 
317
 
 
318
        :param name: The refname to delete.
 
319
        :param old_ref: The old sha the refname must refer to, or None to
 
320
            delete unconditionally.
 
321
        :return: True if the delete was successful, False otherwise.
 
322
        """
 
323
        self._check_refname(name)
 
324
        # may only be packed
 
325
        if name == b'HEAD':
 
326
            transport = self.worktree_transport
 
327
        else:
 
328
            transport = self.transport
 
329
        try:
 
330
            transport.delete(urlutils.quote_from_bytes(name))
 
331
        except NoSuchFile:
 
332
            pass
 
333
        self._remove_packed_ref(name)
 
334
        return True
 
335
 
 
336
    def get(self, name, default=None):
 
337
        try:
 
338
            return self[name]
 
339
        except KeyError:
 
340
            return default
 
341
 
 
342
    def unlock_ref(self, name):
 
343
        if name == b"HEAD":
 
344
            transport = self.worktree_transport
 
345
        else:
 
346
            transport = self.transport
 
347
        lockname = name + b".lock"
 
348
        try:
 
349
            transport.delete(urlutils.quote_from_bytes(lockname))
 
350
        except NoSuchFile:
 
351
            pass
 
352
 
 
353
    def lock_ref(self, name):
 
354
        if name == b"HEAD":
 
355
            transport = self.worktree_transport
 
356
        else:
 
357
            transport = self.transport
 
358
        self._ensure_dir_exists(urlutils.quote_from_bytes(name))
 
359
        lockname = urlutils.quote_from_bytes(name + b".lock")
 
360
        try:
 
361
            local_path = transport.local_abspath(
 
362
                urlutils.quote_from_bytes(name))
 
363
        except NotLocalUrl:
 
364
            # This is racy, but what can we do?
 
365
            if transport.has(lockname):
 
366
                raise LockContention(name)
 
367
            transport.put_bytes(lockname, b'Locked by brz-git')
 
368
            return LogicalLockResult(lambda: transport.delete(lockname))
 
369
        else:
 
370
            try:
 
371
                gf = GitFile(local_path, 'wb')
 
372
            except FileLocked as e:
 
373
                raise LockContention(name, e)
 
374
            else:
 
375
                def unlock():
 
376
                    try:
 
377
                        transport.delete(lockname)
 
378
                    except NoSuchFile:
 
379
                        raise LockBroken(lockname)
 
380
                    # GitFile.abort doesn't care if the lock has already
 
381
                    # disappeared
 
382
                    gf.abort()
 
383
                return LogicalLockResult(unlock)
 
384
 
 
385
 
 
386
# TODO(jelmer): Use upstream read_gitfile; unfortunately that expects strings
 
387
# rather than bytes..
 
388
def read_gitfile(f):
 
389
    """Read a ``.git`` file.
 
390
 
 
391
    The first line of the file should start with "gitdir: "
 
392
 
 
393
    :param f: File-like object to read from
 
394
    :return: A path
 
395
    """
 
396
    cs = f.read()
 
397
    if not cs.startswith(b"gitdir: "):
 
398
        raise ValueError("Expected file to start with 'gitdir: '")
 
399
    return cs[len(b"gitdir: "):].rstrip(b"\n")
 
400
 
 
401
 
 
402
class TransportRepo(BaseRepo):
 
403
 
 
404
    def __init__(self, transport, bare, refs_text=None):
 
405
        self.transport = transport
 
406
        self.bare = bare
 
407
        try:
 
408
            with transport.get(CONTROLDIR) as f:
 
409
                path = read_gitfile(f)
 
410
        except (ReadError, NoSuchFile):
 
411
            if self.bare:
 
412
                self._controltransport = self.transport
 
413
            else:
 
414
                self._controltransport = self.transport.clone('.git')
 
415
        else:
 
416
            self._controltransport = self.transport.clone(
 
417
                urlutils.quote_from_bytes(path))
 
418
        commondir = self.get_named_file(COMMONDIR)
 
419
        if commondir is not None:
 
420
            with commondir:
 
421
                commondir = os.path.join(
 
422
                    self.controldir(),
 
423
                    commondir.read().rstrip(b"\r\n").decode(
 
424
                        sys.getfilesystemencoding()))
 
425
                self._commontransport = \
 
426
                    _mod_transport.get_transport_from_path(commondir)
 
427
        else:
 
428
            self._commontransport = self._controltransport
 
429
        config = self.get_config()
 
430
        object_store = TransportObjectStore.from_config(
 
431
            self._commontransport.clone(OBJECTDIR),
 
432
            config)
 
433
        if refs_text is not None:
 
434
            refs_container = InfoRefsContainer(BytesIO(refs_text))
 
435
            try:
 
436
                head = TransportRefsContainer(
 
437
                    self._commontransport).read_loose_ref(b"HEAD")
 
438
            except KeyError:
 
439
                pass
 
440
            else:
 
441
                refs_container._refs[b"HEAD"] = head
 
442
        else:
 
443
            refs_container = TransportRefsContainer(
 
444
                self._commontransport, self._controltransport)
 
445
        super(TransportRepo, self).__init__(object_store,
 
446
                                            refs_container)
 
447
 
 
448
    def controldir(self):
 
449
        return self._controltransport.local_abspath('.')
 
450
 
 
451
    def commondir(self):
 
452
        return self._commontransport.local_abspath('.')
 
453
 
 
454
    @property
 
455
    def path(self):
 
456
        return self.transport.local_abspath('.')
 
457
 
 
458
    def _determine_file_mode(self):
 
459
        # Be consistent with bzr
 
460
        if sys.platform == 'win32':
 
461
            return False
 
462
        return True
 
463
 
 
464
    def get_named_file(self, path):
 
465
        """Get a file from the control dir with a specific name.
 
466
 
 
467
        Although the filename should be interpreted as a filename relative to
 
468
        the control dir in a disk-baked Repo, the object returned need not be
 
469
        pointing to a file in that location.
 
470
 
 
471
        :param path: The path to the file, relative to the control dir.
 
472
        :return: An open file object, or None if the file does not exist.
 
473
        """
 
474
        try:
 
475
            return self._controltransport.get(path.lstrip('/'))
 
476
        except NoSuchFile:
 
477
            return None
 
478
 
 
479
    def _put_named_file(self, relpath, contents):
 
480
        self._controltransport.put_bytes(relpath, contents)
 
481
 
 
482
    def index_path(self):
 
483
        """Return the path to the index file."""
 
484
        return self._controltransport.local_abspath(INDEX_FILENAME)
 
485
 
 
486
    def open_index(self):
 
487
        """Open the index for this repository."""
 
488
        from dulwich.index import Index
 
489
        if not self.has_index():
 
490
            raise NoIndexPresent()
 
491
        return Index(self.index_path())
 
492
 
 
493
    def has_index(self):
 
494
        """Check if an index is present."""
 
495
        # Bare repos must never have index files; non-bare repos may have a
 
496
        # missing index file, which is treated as empty.
 
497
        return not self.bare
 
498
 
 
499
    def get_config(self):
 
500
        from dulwich.config import ConfigFile
 
501
        try:
 
502
            with self._controltransport.get('config') as f:
 
503
                return ConfigFile.from_file(f)
 
504
        except NoSuchFile:
 
505
            return ConfigFile()
 
506
 
 
507
    def get_config_stack(self):
 
508
        from dulwich.config import StackedConfig
 
509
        backends = []
 
510
        p = self.get_config()
 
511
        if p is not None:
 
512
            backends.append(p)
 
513
            writable = p
 
514
        else:
 
515
            writable = None
 
516
        backends.extend(StackedConfig.default_backends())
 
517
        return StackedConfig(backends, writable=writable)
 
518
 
 
519
    def __repr__(self):
 
520
        return "<%s for %r>" % (self.__class__.__name__, self.transport)
 
521
 
 
522
    @classmethod
 
523
    def init(cls, transport, bare=False):
 
524
        if not bare:
 
525
            try:
 
526
                transport.mkdir(".git")
 
527
            except FileExists:
 
528
                raise AlreadyControlDirError(transport.base)
 
529
            control_transport = transport.clone(".git")
 
530
        else:
 
531
            control_transport = transport
 
532
        for d in BASE_DIRECTORIES:
 
533
            try:
 
534
                control_transport.mkdir("/".join(d))
 
535
            except FileExists:
 
536
                pass
 
537
        try:
 
538
            control_transport.mkdir(OBJECTDIR)
 
539
        except FileExists:
 
540
            raise AlreadyControlDirError(transport.base)
 
541
        TransportObjectStore.init(control_transport.clone(OBJECTDIR))
 
542
        ret = cls(transport, bare)
 
543
        ret.refs.set_symbolic_ref(b"HEAD", b"refs/heads/master")
 
544
        ret._init_files(bare)
 
545
        return ret
 
546
 
 
547
 
 
548
class TransportObjectStore(PackBasedObjectStore):
 
549
    """Git-style object store that exists on disk."""
 
550
 
 
551
    def __init__(self, transport,
 
552
                 loose_compression_level=-1, pack_compression_level=-1):
 
553
        """Open an object store.
 
554
 
 
555
        :param transport: Transport to open data from
 
556
        """
 
557
        super(TransportObjectStore, self).__init__()
 
558
        self.pack_compression_level = pack_compression_level
 
559
        self.loose_compression_level = loose_compression_level
 
560
        self.transport = transport
 
561
        self.pack_transport = self.transport.clone(PACKDIR)
 
562
        self._alternates = None
 
563
 
 
564
    @classmethod
 
565
    def from_config(cls, path, config):
 
566
        try:
 
567
            default_compression_level = int(config.get(
 
568
                (b'core', ), b'compression').decode())
 
569
        except KeyError:
 
570
            default_compression_level = -1
 
571
        try:
 
572
            loose_compression_level = int(config.get(
 
573
                (b'core', ), b'looseCompression').decode())
 
574
        except KeyError:
 
575
            loose_compression_level = default_compression_level
 
576
        try:
 
577
            pack_compression_level = int(config.get(
 
578
                (b'core', ), 'packCompression').decode())
 
579
        except KeyError:
 
580
            pack_compression_level = default_compression_level
 
581
        return cls(path, loose_compression_level, pack_compression_level)
 
582
 
 
583
    def __eq__(self, other):
 
584
        if not isinstance(other, TransportObjectStore):
 
585
            return False
 
586
        return self.transport == other.transport
 
587
 
 
588
    def __repr__(self):
 
589
        return "%s(%r)" % (self.__class__.__name__, self.transport)
 
590
 
 
591
    @property
 
592
    def alternates(self):
 
593
        if self._alternates is not None:
 
594
            return self._alternates
 
595
        self._alternates = []
 
596
        for path in self._read_alternate_paths():
 
597
            # FIXME: Check path
 
598
            t = _mod_transport.get_transport_from_path(path)
 
599
            self._alternates.append(self.__class__(t))
 
600
        return self._alternates
 
601
 
 
602
    def _read_alternate_paths(self):
 
603
        try:
 
604
            f = self.transport.get("info/alternates")
 
605
        except NoSuchFile:
 
606
            return []
 
607
        ret = []
 
608
        with f:
 
609
            for l in f.read().splitlines():
 
610
                if l[0] == b"#":
 
611
                    continue
 
612
                if os.path.isabs(l):
 
613
                    continue
 
614
                ret.append(l)
 
615
            return ret
 
616
 
 
617
    def _update_pack_cache(self):
 
618
        pack_files = set(self._pack_names())
 
619
        new_packs = []
 
620
        for basename in pack_files:
 
621
            pack_name = basename + ".pack"
 
622
            if basename not in self._pack_cache:
 
623
                try:
 
624
                    size = self.pack_transport.stat(pack_name).st_size
 
625
                except TransportNotPossible:
 
626
                    f = self.pack_transport.get(pack_name)
 
627
                    # TODO(jelmer): Don't read entire file into memory?
 
628
                    f = BytesIO(f.read())
 
629
                    pd = PackData(pack_name, f)
 
630
                else:
 
631
                    pd = PackData(
 
632
                        pack_name, self.pack_transport.get(pack_name),
 
633
                        size=size)
 
634
                idxname = basename + ".idx"
 
635
                idx = load_pack_index_file(
 
636
                    idxname, self.pack_transport.get(idxname))
 
637
                pack = Pack.from_objects(pd, idx)
 
638
                pack._basename = basename
 
639
                self._pack_cache[basename] = pack
 
640
                new_packs.append(pack)
 
641
        # Remove disappeared pack files
 
642
        for f in set(self._pack_cache) - pack_files:
 
643
            self._pack_cache.pop(f).close()
 
644
        return new_packs
 
645
 
 
646
    def _pack_names(self):
 
647
        pack_files = []
 
648
        try:
 
649
            dir_contents = self.pack_transport.list_dir(".")
 
650
            for name in dir_contents:
 
651
                if name.startswith("pack-") and name.endswith(".pack"):
 
652
                    # verify that idx exists first (otherwise the pack was not yet
 
653
                    # fully written)
 
654
                    idx_name = os.path.splitext(name)[0] + ".idx"
 
655
                    if idx_name in dir_contents:
 
656
                        pack_files.append(os.path.splitext(name)[0])
 
657
        except TransportNotPossible:
 
658
            try:
 
659
                f = self.transport.get('info/packs')
 
660
            except NoSuchFile:
 
661
                warning('No info/packs on remote host;'
 
662
                        'run \'git update-server-info\' on remote.')
 
663
            else:
 
664
                with f:
 
665
                    pack_files = [
 
666
                        os.path.splitext(name)[0]
 
667
                        for name in read_packs_file(f)]
 
668
        except NoSuchFile:
 
669
            pass
 
670
        return pack_files
 
671
 
 
672
    def _remove_pack(self, pack):
 
673
        self.pack_transport.delete(os.path.basename(pack.index.path))
 
674
        self.pack_transport.delete(pack.data.filename)
 
675
        try:
 
676
            del self._pack_cache[os.path.basename(pack._basename)]
 
677
        except KeyError:
 
678
            pass
 
679
 
 
680
    def _iter_loose_objects(self):
 
681
        for base in self.transport.list_dir('.'):
 
682
            if len(base) != 2:
 
683
                continue
 
684
            for rest in self.transport.list_dir(base):
 
685
                yield (base + rest).encode(sys.getfilesystemencoding())
 
686
 
 
687
    def _split_loose_object(self, sha):
 
688
        return (sha[:2], sha[2:])
 
689
 
 
690
    def _remove_loose_object(self, sha):
 
691
        path = osutils.joinpath(self._split_loose_object(sha))
 
692
        self.transport.delete(urlutils.quote_from_bytes(path))
 
693
 
 
694
    def _get_loose_object(self, sha):
 
695
        path = osutils.joinpath(self._split_loose_object(sha))
 
696
        try:
 
697
            with self.transport.get(urlutils.quote_from_bytes(path)) as f:
 
698
                return ShaFile.from_file(f)
 
699
        except NoSuchFile:
 
700
            return None
 
701
 
 
702
    def add_object(self, obj):
 
703
        """Add a single object to this object store.
 
704
 
 
705
        :param obj: Object to add
 
706
        """
 
707
        (dir, file) = self._split_loose_object(obj.id)
 
708
        try:
 
709
            self.transport.mkdir(urlutils.quote_from_bytes(dir))
 
710
        except FileExists:
 
711
            pass
 
712
        path = urlutils.quote_from_bytes(osutils.pathjoin(dir, file))
 
713
        if self.transport.has(path):
 
714
            return  # Already there, no need to write again
 
715
        # Backwards compatibility with Dulwich < 0.20, which doesn't support
 
716
        # the compression_level parameter.
 
717
        if self.loose_compression_level not in (-1, None):
 
718
            raw_string = obj.as_legacy_object(
 
719
                compression_level=self.loose_compression_level)
 
720
        else:
 
721
            raw_string = obj.as_legacy_object()
 
722
        self.transport.put_bytes(path, raw_string)
 
723
 
 
724
    def move_in_pack(self, f):
 
725
        """Move a specific file containing a pack into the pack directory.
 
726
 
 
727
        :note: The file should be on the same file system as the
 
728
            packs directory.
 
729
 
 
730
        :param path: Path to the pack file.
 
731
        """
 
732
        f.seek(0)
 
733
        p = PackData("", f, len(f.getvalue()))
 
734
        entries = p.sorted_entries()
 
735
        basename = "pack-%s" % iter_sha1(entry[0]
 
736
                                         for entry in entries).decode('ascii')
 
737
        p._filename = basename + ".pack"
 
738
        f.seek(0)
 
739
        self.pack_transport.put_file(basename + ".pack", f)
 
740
        with self.pack_transport.open_write_stream(basename + ".idx") as idxfile:
 
741
            write_pack_index_v2(idxfile, entries, p.get_stored_checksum())
 
742
        idxfile = self.pack_transport.get(basename + ".idx")
 
743
        idx = load_pack_index_file(basename + ".idx", idxfile)
 
744
        final_pack = Pack.from_objects(p, idx)
 
745
        final_pack._basename = basename
 
746
        self._add_cached_pack(basename, final_pack)
 
747
        return final_pack
 
748
 
 
749
    def move_in_thin_pack(self, f):
 
750
        """Move a specific file containing a pack into the pack directory.
 
751
 
 
752
        :note: The file should be on the same file system as the
 
753
            packs directory.
 
754
 
 
755
        :param path: Path to the pack file.
 
756
        """
 
757
        f.seek(0)
 
758
        p = Pack('', resolve_ext_ref=self.get_raw)
 
759
        p._data = PackData.from_file(f, len(f.getvalue()))
 
760
        p._data.pack = p
 
761
        p._idx_load = lambda: MemoryPackIndex(
 
762
            p.data.sorted_entries(), p.data.get_stored_checksum())
 
763
 
 
764
        pack_sha = p.index.objects_sha1()
 
765
 
 
766
        with self.pack_transport.open_write_stream(
 
767
                "pack-%s.pack" % pack_sha.decode('ascii')) as datafile:
 
768
            entries, data_sum = write_pack_objects(datafile, p.pack_tuples())
 
769
        entries = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
 
770
        with self.pack_transport.open_write_stream(
 
771
                "pack-%s.idx" % pack_sha.decode('ascii')) as idxfile:
 
772
            write_pack_index_v2(idxfile, entries, data_sum)
 
773
 
 
774
    def add_pack(self):
 
775
        """Add a new pack to this object store.
 
776
 
 
777
        :return: Fileobject to write to and a commit function to
 
778
            call when the pack is finished.
 
779
        """
 
780
        f = BytesIO()
 
781
 
 
782
        def commit():
 
783
            if len(f.getvalue()) > 0:
 
784
                return self.move_in_pack(f)
 
785
            else:
 
786
                return None
 
787
 
 
788
        def abort():
 
789
            return None
 
790
        return f, commit, abort
 
791
 
 
792
    @classmethod
 
793
    def init(cls, transport):
 
794
        try:
 
795
            transport.mkdir('info')
 
796
        except FileExists:
 
797
            pass
 
798
        try:
 
799
            transport.mkdir(PACKDIR)
 
800
        except FileExists:
 
801
            pass
 
802
        return cls(transport)