1
# Copyright (C) 2007-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Remote dirs, repositories and branches."""
19
from __future__ import absolute_import
28
from ...errors import (
38
UninitializableFormat,
40
from ...transport import (
57
BareLocalGitControlDirFormat,
60
GitSmartRemoteNotSupported,
63
from .mapping import (
66
from .repository import (
78
from dulwich.errors import (
81
from dulwich.pack import (
85
from dulwich.refs import SYMREF
86
from dulwich.repo import DictRefsContainer
93
# urlparse only supports a limited number of schemes by default
95
urlparse.uses_netloc.extend(['git', 'git+ssh'])
97
from dulwich.pack import load_pack_index
100
# Don't run any tests on GitSmartTransport as it is not intended to be
101
# a full implementation of Transport
102
def get_test_permutations():
106
def split_git_url(url):
110
:return: Tuple with host, port, username, path.
112
(scheme, netloc, loc, _, _) = urlparse.urlsplit(url)
113
path = urllib.unquote(loc)
114
if path.startswith("/~"):
116
(username, hostport) = urllib.splituser(netloc)
117
(host, port) = urllib.splitnport(hostport, None)
118
return (host, port, username, path)
121
class RemoteGitError(BzrError):
123
_fmt = "Remote server error: %(msg)s"
126
def parse_git_error(url, message):
127
"""Parse a remote git server error and return a bzr exception.
129
:param url: URL of the remote repository
130
:param message: Message sent by the remote git server
132
message = str(message).strip()
133
if message.startswith("Could not find Repository "):
134
return NotBranchError(url, message)
135
if message == "HEAD failed to update":
136
base_url, _ = urlutils.split_segment_parameters(url)
138
("Unable to update remote HEAD branch. To update the master "
139
"branch, specify the URL %s,branch=master.") % base_url)
140
# Don't know, just return it to the user as-is
141
return RemoteGitError(message)
144
class GitSmartTransport(Transport):
146
def __init__(self, url, _client=None):
147
Transport.__init__(self, url)
148
(self._host, self._port, self._username, self._path) = \
150
if 'transport' in debug.debug_flags:
151
trace.mutter('host: %r, user: %r, port: %r, path: %r',
152
self._host, self._username, self._port, self._path)
153
self._client = _client
154
self._stripped_path = self._path.rsplit(",", 1)[0]
156
def external_url(self):
159
def has(self, relpath):
162
def _get_client(self):
163
raise NotImplementedError(self._get_client)
166
return self._stripped_path
169
raise NoSuchFile(path)
171
def abspath(self, relpath):
172
return urlutils.join(self.base, relpath)
174
def clone(self, offset=None):
175
"""See Transport.clone()."""
179
newurl = urlutils.join(self.base, offset)
181
return self.__class__(newurl, self._client)
184
class TCPGitSmartTransport(GitSmartTransport):
188
def _get_client(self):
189
if self._client is not None:
194
# return dulwich.client.LocalGitClient()
195
return dulwich.client.SubprocessGitClient()
196
return dulwich.client.TCPGitClient(self._host, self._port,
197
report_activity=self._report_activity)
200
class SSHSocketWrapper(object):
202
def __init__(self, sock):
205
def read(self, len=None):
206
return self.sock.recv(len)
208
def write(self, data):
209
return self.sock.write(data)
212
return len(select.select([self.sock.fileno()], [], [], 0)[0]) > 0
215
class DulwichSSHVendor(dulwich.client.SSHVendor):
218
from ...transport import ssh
219
self.bzr_ssh_vendor = ssh._get_ssh_vendor()
221
def run_command(self, host, command, username=None, port=None):
222
connection = self.bzr_ssh_vendor.connect_ssh(username=username,
223
password=None, port=port, host=host, command=command)
224
(kind, io_object) = connection.get_sock_or_pipes()
226
return SSHSocketWrapper(io_object)
228
raise AssertionError("Unknown io object kind %r'" % kind)
231
#dulwich.client.get_ssh_vendor = DulwichSSHVendor
234
class SSHGitSmartTransport(GitSmartTransport):
239
path = self._stripped_path
240
if path.startswith("/~/"):
244
def _get_client(self):
245
if self._client is not None:
249
location_config = config.LocationConfig(self.base)
250
client = dulwich.client.SSHGitClient(self._host, self._port, self._username,
251
report_activity=self._report_activity)
252
# Set up alternate pack program paths
253
upload_pack = location_config.get_user_option('git_upload_pack')
255
client.alternative_paths["upload-pack"] = upload_pack
256
receive_pack = location_config.get_user_option('git_receive_pack')
258
client.alternative_paths["receive-pack"] = receive_pack
262
class RemoteGitBranchFormat(GitBranchFormat):
264
def get_format_description(self):
265
return 'Remote Git Branch'
268
def _matchingcontroldir(self):
269
return RemoteGitControlDirFormat()
271
def initialize(self, a_controldir, name=None, repository=None,
272
append_revisions_only=None):
273
raise UninitializableFormat(self)
276
def default_report_progress(text):
277
if text.startswith('error: '):
278
trace.show_error('git: %s', text[len('error: '):])
280
trace.mutter("git: %s" % text)
283
class RemoteGitDir(GitDir):
285
def __init__(self, transport, format, client, client_path):
286
self._format = format
287
self.root_transport = transport
288
self.transport = transport
289
self._mode_check_done = None
290
self._client = client
291
self._client_path = client_path
292
self.base = self.root_transport.base
296
def _gitrepository_class(self):
297
return RemoteGitRepository
299
def fetch_pack(self, determine_wants, graph_walker, pack_data, progress=None):
301
progress = default_report_progress
303
result = self._client.fetch_pack(self._client_path, determine_wants,
304
graph_walker, pack_data, progress)
305
if result.refs is None:
307
self._refs = remote_refs_dict_to_container(result.refs, result.symrefs)
309
except GitProtocolError, e:
310
raise parse_git_error(self.transport.external_url(), e)
312
def send_pack(self, get_changed_refs, generate_pack_data, progress=None):
314
progress = default_report_progress
317
return self._client.send_pack(self._client_path, get_changed_refs,
318
generate_pack_data, progress)
319
except GitProtocolError, e:
320
raise parse_git_error(self.transport.external_url(), e)
322
def create_branch(self, name=None, repository=None,
323
append_revisions_only=None, ref=None):
324
refname = self._get_selected_ref(name, ref)
325
if refname != b'HEAD' and refname in self.get_refs_container():
326
raise AlreadyBranchError(self.user_url)
327
if refname in self.get_refs_container():
328
ref_chain, unused_sha = self.get_refs_container().follow(self._get_selected_ref(None))
329
if ref_chain[0] == b'HEAD':
330
refname = ref_chain[1]
331
repo = self.open_repository()
332
return RemoteGitBranch(self, repo, refname)
334
def destroy_branch(self, name=None):
335
refname = self._get_selected_ref(name)
336
def get_changed_refs(old_refs):
338
if not refname in ret:
339
raise NotBranchError(self.user_url)
340
ret[refname] = dulwich.client.ZERO_SHA
342
def generate_pack_data(have, want, ofs_delta=False):
343
return pack_objects_to_data([])
344
self.send_pack(get_changed_refs, generate_pack_data)
348
return self.control_url
351
def user_transport(self):
352
return self.root_transport
355
def control_url(self):
356
return self.control_transport.base
359
def control_transport(self):
360
return self.root_transport
362
def open_repository(self):
363
return RemoteGitRepository(self)
365
def open_branch(self, name=None, unsupported=False,
366
ignore_fallbacks=False, ref=None, possible_transports=None,
368
repo = self.open_repository()
369
ref = self._get_selected_ref(name, ref)
370
if not nascent_ok and ref not in self.get_refs_container():
371
raise NotBranchError(self.root_transport.base,
373
ref_chain, unused_sha = self.get_refs_container().follow(ref)
374
return RemoteGitBranch(self, repo, ref_chain[-1])
376
def open_workingtree(self, recommend_upgrade=False):
377
raise NotLocalUrl(self.transport.base)
379
def has_workingtree(self):
382
def get_peeled(self, name):
383
return self.get_refs_container().get_peeled(name)
385
def get_refs_container(self):
386
if self._refs is not None:
388
result = self.fetch_pack(lambda x: None, None,
389
lambda x: None, lambda x: trace.mutter("git: %s" % x))
390
self._refs = remote_refs_dict_to_container(
391
result.refs, result.symrefs)
395
class EmptyObjectStoreIterator(dict):
397
def iterobjects(self):
401
class TemporaryPackIterator(Pack):
403
def __init__(self, path, resolve_ext_ref):
404
super(TemporaryPackIterator, self).__init__(
405
path, resolve_ext_ref=resolve_ext_ref)
406
self._idx_load = lambda: self._idx_load_or_generate(self._idx_path)
408
def _idx_load_or_generate(self, path):
409
if not os.path.exists(path):
410
pb = ui.ui_factory.nested_progress_bar()
412
def report_progress(cur, total):
413
pb.update("generating index", cur, total)
414
self.data.create_index(path,
415
progress=report_progress)
418
return load_pack_index(path)
421
if self._idx is not None:
423
os.remove(self._idx_path)
424
if self._data is not None:
426
os.remove(self._data_path)
429
class BzrGitHttpClient(dulwich.client.HttpGitClient):
431
def __init__(self, transport, *args, **kwargs):
432
self.transport = transport
433
super(BzrGitHttpClient, self).__init__(transport.external_url(), *args, **kwargs)
435
self._http_perform = getattr(self.transport, "_perform", urllib2.urlopen)
437
def _perform(self, req):
438
req.accepted_errors = (200, 404)
439
req.follow_redirections = True
440
req.redirected_to = None
441
return self._http_perform(req)
444
class RemoteGitControlDirFormat(GitControlDirFormat):
445
"""The .git directory control format."""
447
supports_workingtrees = False
450
def _known_formats(self):
451
return set([RemoteGitControlDirFormat()])
453
def get_branch_format(self):
454
return RemoteGitBranchFormat()
456
def is_initializable(self):
459
def is_supported(self):
462
def open(self, transport, _found=None):
463
"""Open this directory.
466
# we dont grok readonly - git isn't integrated with transport.
468
if url.startswith('readonly+'):
469
url = url[len('readonly+'):]
470
scheme = urlparse.urlsplit(transport.external_url())[0]
471
if isinstance(transport, GitSmartTransport):
472
client = transport._get_client()
473
client_path = transport._get_path()
474
elif scheme in ("http", "https"):
475
client = BzrGitHttpClient(transport)
476
client_path, _ = urlutils.split_segment_parameters(transport._path)
477
elif scheme == 'file':
478
client = dulwich.client.LocalGitClient()
479
client_path = transport.local_abspath('.')
481
raise NotBranchError(transport.base)
483
pass # TODO(jelmer): Actually probe for something
484
return RemoteGitDir(transport, self, client, client_path)
486
def get_format_description(self):
487
return "Remote Git Repository"
489
def initialize_on_transport(self, transport):
490
raise UninitializableFormat(self)
492
def supports_transport(self, transport):
494
external_url = transport.external_url()
495
except InProcessTransport:
496
raise NotBranchError(path=transport.base)
497
return (external_url.startswith("http:") or
498
external_url.startswith("https:") or
499
external_url.startswith("git+") or
500
external_url.startswith("git:"))
503
class RemoteGitRepository(GitRepository):
507
return self.control_url
509
def get_parent_map(self, revids):
510
raise GitSmartRemoteNotSupported(self.get_parent_map, self)
512
def fetch_pack(self, determine_wants, graph_walker, pack_data,
514
return self.controldir.fetch_pack(determine_wants, graph_walker,
517
def send_pack(self, get_changed_refs, generate_pack_data):
518
return self.controldir.send_pack(get_changed_refs, generate_pack_data)
520
def fetch_objects(self, determine_wants, graph_walker, resolve_ext_ref,
522
fd, path = tempfile.mkstemp(suffix=".pack")
524
self.fetch_pack(determine_wants, graph_walker,
525
lambda x: os.write(fd, x), progress)
528
if os.path.getsize(path) == 0:
529
return EmptyObjectStoreIterator()
530
return TemporaryPackIterator(path[:-len(".pack")], resolve_ext_ref)
532
def lookup_bzr_revision_id(self, bzr_revid, mapping=None):
533
# This won't work for any round-tripped bzr revisions, but it's a start..
535
return mapping_registry.revision_id_bzr_to_foreign(bzr_revid)
536
except InvalidRevisionId:
537
raise NoSuchRevision(self, bzr_revid)
539
def lookup_foreign_revision_id(self, foreign_revid, mapping=None):
540
"""Lookup a revision id.
544
mapping = self.get_mapping()
545
# Not really an easy way to parse foreign revids here..
546
return mapping.revision_id_foreign_to_bzr(foreign_revid)
548
def revision_tree(self, revid):
549
raise GitSmartRemoteNotSupported(self.revision_tree, self)
551
def get_revisions(self, revids):
552
raise GitSmartRemoteNotSupported(self.get_revisions, self)
554
def has_revisions(self, revids):
555
raise GitSmartRemoteNotSupported(self.get_revisions, self)
558
class RemoteGitTagDict(GitTags):
560
def set_tag(self, name, revid):
561
sha = self.branch.lookup_bzr_revision_id(revid)[0]
562
self._set_ref(name, sha)
564
def delete_tag(self, name):
565
self._set_ref(name, dulwich.client.ZERO_SHA)
567
def _set_ref(self, name, sha):
568
ref = tag_name_to_ref(name)
569
def get_changed_refs(old_refs):
571
if sha == dulwich.client.ZERO_SHA and ref not in ret:
572
raise NoSuchTag(name)
575
def generate_pack_data(have, want, ofs_delta=False):
576
return pack_objects_to_data([])
577
self.repository.send_pack(get_changed_refs, generate_pack_data)
580
class RemoteGitBranch(GitBranch):
582
def __init__(self, controldir, repository, name):
584
super(RemoteGitBranch, self).__init__(controldir, repository, name,
585
RemoteGitBranchFormat())
587
def last_revision_info(self):
588
raise GitSmartRemoteNotSupported(self.last_revision_info, self)
592
return self.control_url
595
def control_url(self):
598
def revision_id_to_revno(self, revision_id):
599
raise GitSmartRemoteNotSupported(self.revision_id_to_revno, self)
601
def last_revision(self):
602
return self.lookup_foreign_revision_id(self.head)
606
if self._sha is not None:
608
refs = self.controldir.get_refs_container()
609
name = branch_name_to_ref(self.name)
611
self._sha = refs[name]
613
raise NoSuchRef(name, self.repository.user_url, refs)
616
def _synchronize_history(self, destination, revision_id):
617
"""See Branch._synchronize_history()."""
618
destination.generate_revision_history(self.last_revision())
620
def _get_parent_location(self):
623
def get_push_location(self):
626
def set_push_location(self, url):
629
def _iter_tag_refs(self):
630
"""Iterate over the tag refs.
632
:param refs: Refs dictionary (name -> git sha1)
633
:return: iterator over (ref_name, tag_name, peeled_sha1, unpeeled_sha1)
635
refs = self.controldir.get_refs_container()
636
for ref_name, unpeeled in refs.as_dict().iteritems():
638
tag_name = ref_to_tag_name(ref_name)
639
except (ValueError, UnicodeDecodeError):
641
peeled = refs.get_peeled(ref_name)
644
peeled = refs.peel_sha(unpeeled).id
646
# Let's just hope it's a commit
648
if type(tag_name) is not unicode:
649
raise TypeError(tag_name)
650
yield (ref_name, tag_name, peeled, unpeeled)
653
def remote_refs_dict_to_container(refs_dict, symrefs_dict={}):
656
for k, v in refs_dict.iteritems():
662
for name, target in symrefs_dict.iteritems():
663
base[name] = SYMREF + target
664
ret = DictRefsContainer(base)