1
# Copyright (C) 2007-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Remote dirs, repositories and branches."""
19
from __future__ import absolute_import
31
from ...errors import (
42
UninitializableFormat,
44
from ...transport import (
62
BareLocalGitControlDirFormat,
65
GitSmartRemoteNotSupported,
68
from .mapping import (
71
from .object_store import (
74
from .repository import (
86
from dulwich.errors import (
89
from dulwich.pack import (
93
from dulwich.protocol import ZERO_SHA
94
from dulwich.refs import SYMREF
95
from dulwich.repo import DictRefsContainer
102
# urlparse only supports a limited number of schemes by default
104
urlparse.uses_netloc.extend(['git', 'git+ssh'])
106
from dulwich.pack import load_pack_index
109
# Don't run any tests on GitSmartTransport as it is not intended to be
110
# a full implementation of Transport
111
def get_test_permutations():
115
def split_git_url(url):
119
:return: Tuple with host, port, username, path.
121
(scheme, netloc, loc, _, _) = urlparse.urlsplit(url)
122
path = urllib.unquote(loc)
123
if path.startswith("/~"):
125
(username, hostport) = urllib.splituser(netloc)
126
(host, port) = urllib.splitnport(hostport, None)
127
return (host, port, username, path)
130
class RemoteGitError(BzrError):
132
_fmt = "Remote server error: %(msg)s"
135
def parse_git_error(url, message):
136
"""Parse a remote git server error and return a bzr exception.
138
:param url: URL of the remote repository
139
:param message: Message sent by the remote git server
141
message = str(message).strip()
142
if message.startswith("Could not find Repository "):
143
return NotBranchError(url, message)
144
if message == "HEAD failed to update":
145
base_url, _ = urlutils.split_segment_parameters(url)
147
("Unable to update remote HEAD branch. To update the master "
148
"branch, specify the URL %s,branch=master.") % base_url)
149
# Don't know, just return it to the user as-is
150
return RemoteGitError(message)
153
class GitSmartTransport(Transport):
155
def __init__(self, url, _client=None):
156
Transport.__init__(self, url)
157
(self._host, self._port, self._username, self._path) = \
159
if 'transport' in debug.debug_flags:
160
trace.mutter('host: %r, user: %r, port: %r, path: %r',
161
self._host, self._username, self._port, self._path)
162
self._client = _client
163
self._stripped_path = self._path.rsplit(",", 1)[0]
165
def external_url(self):
168
def has(self, relpath):
171
def _get_client(self):
172
raise NotImplementedError(self._get_client)
175
return self._stripped_path
178
raise NoSuchFile(path)
180
def abspath(self, relpath):
181
return urlutils.join(self.base, relpath)
183
def clone(self, offset=None):
184
"""See Transport.clone()."""
188
newurl = urlutils.join(self.base, offset)
190
return self.__class__(newurl, self._client)
193
class TCPGitSmartTransport(GitSmartTransport):
197
def _get_client(self):
198
if self._client is not None:
203
# return dulwich.client.LocalGitClient()
204
return dulwich.client.SubprocessGitClient()
205
return dulwich.client.TCPGitClient(self._host, self._port,
206
report_activity=self._report_activity)
209
class SSHSocketWrapper(object):
211
def __init__(self, sock):
214
def read(self, len=None):
215
return self.sock.recv(len)
217
def write(self, data):
218
return self.sock.write(data)
221
return len(select.select([self.sock.fileno()], [], [], 0)[0]) > 0
224
class DulwichSSHVendor(dulwich.client.SSHVendor):
227
from ...transport import ssh
228
self.bzr_ssh_vendor = ssh._get_ssh_vendor()
230
def run_command(self, host, command, username=None, port=None):
231
connection = self.bzr_ssh_vendor.connect_ssh(username=username,
232
password=None, port=port, host=host, command=command)
233
(kind, io_object) = connection.get_sock_or_pipes()
235
return SSHSocketWrapper(io_object)
237
raise AssertionError("Unknown io object kind %r'" % kind)
240
#dulwich.client.get_ssh_vendor = DulwichSSHVendor
243
class SSHGitSmartTransport(GitSmartTransport):
248
path = self._stripped_path
249
if path.startswith("/~/"):
253
def _get_client(self):
254
if self._client is not None:
258
location_config = config.LocationConfig(self.base)
259
client = dulwich.client.SSHGitClient(self._host, self._port, self._username,
260
report_activity=self._report_activity)
261
# Set up alternate pack program paths
262
upload_pack = location_config.get_user_option('git_upload_pack')
264
client.alternative_paths["upload-pack"] = upload_pack
265
receive_pack = location_config.get_user_option('git_receive_pack')
267
client.alternative_paths["receive-pack"] = receive_pack
271
class RemoteGitBranchFormat(GitBranchFormat):
273
def get_format_description(self):
274
return 'Remote Git Branch'
277
def _matchingcontroldir(self):
278
return RemoteGitControlDirFormat()
280
def initialize(self, a_controldir, name=None, repository=None,
281
append_revisions_only=None):
282
raise UninitializableFormat(self)
285
def default_report_progress(text):
286
if text.startswith('error: '):
287
trace.show_error('git: %s', text[len('error: '):])
289
trace.mutter("git: %s" % text)
292
class RemoteGitDir(GitDir):
294
def __init__(self, transport, format, client, client_path):
295
self._format = format
296
self.root_transport = transport
297
self.transport = transport
298
self._mode_check_done = None
299
self._client = client
300
self._client_path = client_path
301
self.base = self.root_transport.base
305
def _gitrepository_class(self):
306
return RemoteGitRepository
308
def fetch_pack(self, determine_wants, graph_walker, pack_data, progress=None):
310
progress = default_report_progress
312
result = self._client.fetch_pack(self._client_path, determine_wants,
313
graph_walker, pack_data, progress)
314
if result.refs is None:
316
self._refs = remote_refs_dict_to_container(result.refs, result.symrefs)
318
except GitProtocolError, e:
319
raise parse_git_error(self.transport.external_url(), e)
321
def send_pack(self, get_changed_refs, generate_pack_data, progress=None):
323
progress = default_report_progress
326
return self._client.send_pack(self._client_path, get_changed_refs,
327
generate_pack_data, progress)
328
except GitProtocolError, e:
329
raise parse_git_error(self.transport.external_url(), e)
331
def create_branch(self, name=None, repository=None,
332
append_revisions_only=None, ref=None):
333
refname = self._get_selected_ref(name, ref)
334
if refname != b'HEAD' and refname in self.get_refs_container():
335
raise AlreadyBranchError(self.user_url)
336
if refname in self.get_refs_container():
337
ref_chain, unused_sha = self.get_refs_container().follow(self._get_selected_ref(None))
338
if ref_chain[0] == b'HEAD':
339
refname = ref_chain[1]
340
repo = self.open_repository()
341
return RemoteGitBranch(self, repo, refname)
343
def destroy_branch(self, name=None):
344
refname = self._get_selected_ref(name)
345
def get_changed_refs(old_refs):
347
if not refname in ret:
348
raise NotBranchError(self.user_url)
349
ret[refname] = dulwich.client.ZERO_SHA
351
def generate_pack_data(have, want, ofs_delta=False):
352
return pack_objects_to_data([])
353
self.send_pack(get_changed_refs, generate_pack_data)
357
return self.control_url
360
def user_transport(self):
361
return self.root_transport
364
def control_url(self):
365
return self.control_transport.base
368
def control_transport(self):
369
return self.root_transport
371
def open_repository(self):
372
return RemoteGitRepository(self)
374
def open_branch(self, name=None, unsupported=False,
375
ignore_fallbacks=False, ref=None, possible_transports=None,
377
repo = self.open_repository()
378
ref = self._get_selected_ref(name, ref)
379
if not nascent_ok and ref not in self.get_refs_container():
380
raise NotBranchError(self.root_transport.base,
382
ref_chain, unused_sha = self.get_refs_container().follow(ref)
383
return RemoteGitBranch(self, repo, ref_chain[-1])
385
def open_workingtree(self, recommend_upgrade=False):
386
raise NotLocalUrl(self.transport.base)
388
def has_workingtree(self):
391
def get_peeled(self, name):
392
return self.get_refs_container().get_peeled(name)
394
def get_refs_container(self):
395
if self._refs is not None:
397
result = self.fetch_pack(lambda x: None, None,
398
lambda x: None, lambda x: trace.mutter("git: %s" % x))
399
self._refs = remote_refs_dict_to_container(
400
result.refs, result.symrefs)
403
def push_branch(self, source, revision_id=None, overwrite=False,
404
remember=False, create_prefix=False, lossy=False,
406
"""Push the source branch into this ControlDir."""
407
if revision_id is None:
408
# No revision supplied by the user, default to the branch
410
revision_id = source.last_revision()
412
push_result = PushResult()
413
push_result.workingtree_updated = None
414
push_result.master_branch = None
415
push_result.source_branch = source
416
push_result.stacked_on = None
417
push_result.branch_push_result = None
418
repo = self.find_repository()
419
refname = self._get_selected_ref(name)
420
source_store = get_object_store(source.repository)
421
with source_store.lock_read():
422
def get_changed_refs(refs):
423
self._refs = remote_refs_dict_to_container(refs)
425
# TODO(jelmer): Unpeel if necessary
427
ret[refname] = source_store._lookup_revision_sha1(revision_id)
429
ret[refname] = repo.lookup_bzr_revision_id(revision_id)[0]
432
generate_pack_data = source_store.generate_lossy_pack_data
434
generate_pack_data = source_store.generate_pack_data
435
new_refs = self.send_pack(get_changed_refs, generate_pack_data)
436
push_result.new_revid = repo.lookup_foreign_revision_id(
439
old_remote = self._refs[refname]
441
old_remote = ZERO_SHA
442
push_result.old_revid = repo.lookup_foreign_revision_id(old_remote)
443
self._refs = remote_refs_dict_to_container(new_refs)
444
push_result.old_revno = None
445
push_result.target_branch = self.open_branch(name)
446
if old_remote != ZERO_SHA:
447
push_result.branch_push_result = GitBranchPushResult()
448
push_result.branch_push_result.source_branch = source
449
push_result.branch_push_result.target_branch = push_result.target_branch
450
push_result.branch_push_result.local_branch = None
451
push_result.branch_push_result.master_branch = push_result.target_branch
452
push_result.branch_push_result.old_revid = push_result.old_revid
453
push_result.branch_push_result.new_revid = push_result.new_revid
454
if source.get_push_location() is None or remember:
455
source.set_push_location(push_result.target_branch.base)
459
class EmptyObjectStoreIterator(dict):
461
def iterobjects(self):
465
class TemporaryPackIterator(Pack):
467
def __init__(self, path, resolve_ext_ref):
468
super(TemporaryPackIterator, self).__init__(
469
path, resolve_ext_ref=resolve_ext_ref)
470
self._idx_load = lambda: self._idx_load_or_generate(self._idx_path)
472
def _idx_load_or_generate(self, path):
473
if not os.path.exists(path):
474
pb = ui.ui_factory.nested_progress_bar()
476
def report_progress(cur, total):
477
pb.update("generating index", cur, total)
478
self.data.create_index(path,
479
progress=report_progress)
482
return load_pack_index(path)
485
if self._idx is not None:
487
os.remove(self._idx_path)
488
if self._data is not None:
490
os.remove(self._data_path)
493
class BzrGitHttpClient(dulwich.client.HttpGitClient):
495
def __init__(self, transport, *args, **kwargs):
496
self.transport = transport
497
super(BzrGitHttpClient, self).__init__(transport.external_url(), *args, **kwargs)
499
self._http_perform = getattr(self.transport, "_perform", urllib2.urlopen)
501
def _perform(self, req):
502
req.accepted_errors = (200, 404)
503
req.follow_redirections = True
504
req.redirected_to = None
505
return self._http_perform(req)
508
class RemoteGitControlDirFormat(GitControlDirFormat):
509
"""The .git directory control format."""
511
supports_workingtrees = False
514
def _known_formats(self):
515
return set([RemoteGitControlDirFormat()])
517
def get_branch_format(self):
518
return RemoteGitBranchFormat()
520
def is_initializable(self):
523
def is_supported(self):
526
def open(self, transport, _found=None):
527
"""Open this directory.
530
# we dont grok readonly - git isn't integrated with transport.
532
if url.startswith('readonly+'):
533
url = url[len('readonly+'):]
534
scheme = urlparse.urlsplit(transport.external_url())[0]
535
if isinstance(transport, GitSmartTransport):
536
client = transport._get_client()
537
client_path = transport._get_path()
538
elif scheme in ("http", "https"):
539
client = BzrGitHttpClient(transport)
540
client_path, _ = urlutils.split_segment_parameters(transport._path)
541
elif scheme == 'file':
542
client = dulwich.client.LocalGitClient()
543
client_path = transport.local_abspath('.')
545
raise NotBranchError(transport.base)
547
pass # TODO(jelmer): Actually probe for something
548
return RemoteGitDir(transport, self, client, client_path)
550
def get_format_description(self):
551
return "Remote Git Repository"
553
def initialize_on_transport(self, transport):
554
raise UninitializableFormat(self)
556
def supports_transport(self, transport):
558
external_url = transport.external_url()
559
except InProcessTransport:
560
raise NotBranchError(path=transport.base)
561
return (external_url.startswith("http:") or
562
external_url.startswith("https:") or
563
external_url.startswith("git+") or
564
external_url.startswith("git:"))
567
class RemoteGitRepository(GitRepository):
571
return self.control_url
573
def get_parent_map(self, revids):
574
raise GitSmartRemoteNotSupported(self.get_parent_map, self)
576
def fetch_pack(self, determine_wants, graph_walker, pack_data,
578
return self.controldir.fetch_pack(determine_wants, graph_walker,
581
def send_pack(self, get_changed_refs, generate_pack_data):
582
return self.controldir.send_pack(get_changed_refs, generate_pack_data)
584
def fetch_objects(self, determine_wants, graph_walker, resolve_ext_ref,
586
fd, path = tempfile.mkstemp(suffix=".pack")
588
self.fetch_pack(determine_wants, graph_walker,
589
lambda x: os.write(fd, x), progress)
592
if os.path.getsize(path) == 0:
593
return EmptyObjectStoreIterator()
594
return TemporaryPackIterator(path[:-len(".pack")], resolve_ext_ref)
596
def lookup_bzr_revision_id(self, bzr_revid, mapping=None):
597
# This won't work for any round-tripped bzr revisions, but it's a start..
599
return mapping_registry.revision_id_bzr_to_foreign(bzr_revid)
600
except InvalidRevisionId:
601
raise NoSuchRevision(self, bzr_revid)
603
def lookup_foreign_revision_id(self, foreign_revid, mapping=None):
604
"""Lookup a revision id.
608
mapping = self.get_mapping()
609
# Not really an easy way to parse foreign revids here..
610
return mapping.revision_id_foreign_to_bzr(foreign_revid)
612
def revision_tree(self, revid):
613
raise GitSmartRemoteNotSupported(self.revision_tree, self)
615
def get_revisions(self, revids):
616
raise GitSmartRemoteNotSupported(self.get_revisions, self)
618
def has_revisions(self, revids):
619
raise GitSmartRemoteNotSupported(self.get_revisions, self)
622
class RemoteGitTagDict(GitTags):
624
def set_tag(self, name, revid):
625
sha = self.branch.lookup_bzr_revision_id(revid)[0]
626
self._set_ref(name, sha)
628
def delete_tag(self, name):
629
self._set_ref(name, dulwich.client.ZERO_SHA)
631
def _set_ref(self, name, sha):
632
ref = tag_name_to_ref(name)
633
def get_changed_refs(old_refs):
635
if sha == dulwich.client.ZERO_SHA and ref not in ret:
636
raise NoSuchTag(name)
639
def generate_pack_data(have, want, ofs_delta=False):
640
return pack_objects_to_data([])
641
self.repository.send_pack(get_changed_refs, generate_pack_data)
644
class RemoteGitBranch(GitBranch):
646
def __init__(self, controldir, repository, name):
648
super(RemoteGitBranch, self).__init__(controldir, repository, name,
649
RemoteGitBranchFormat())
651
def last_revision_info(self):
652
raise GitSmartRemoteNotSupported(self.last_revision_info, self)
656
return self.control_url
659
def control_url(self):
662
def revision_id_to_revno(self, revision_id):
663
raise GitSmartRemoteNotSupported(self.revision_id_to_revno, self)
665
def last_revision(self):
666
return self.lookup_foreign_revision_id(self.head)
670
if self._sha is not None:
672
refs = self.controldir.get_refs_container()
673
name = branch_name_to_ref(self.name)
675
self._sha = refs[name]
677
raise NoSuchRef(name, self.repository.user_url, refs)
680
def _synchronize_history(self, destination, revision_id):
681
"""See Branch._synchronize_history()."""
682
destination.generate_revision_history(self.last_revision())
684
def _get_parent_location(self):
687
def get_push_location(self):
690
def set_push_location(self, url):
693
def _iter_tag_refs(self):
694
"""Iterate over the tag refs.
696
:param refs: Refs dictionary (name -> git sha1)
697
:return: iterator over (ref_name, tag_name, peeled_sha1, unpeeled_sha1)
699
refs = self.controldir.get_refs_container()
700
for ref_name, unpeeled in refs.as_dict().iteritems():
702
tag_name = ref_to_tag_name(ref_name)
703
except (ValueError, UnicodeDecodeError):
705
peeled = refs.get_peeled(ref_name)
708
peeled = refs.peel_sha(unpeeled).id
710
# Let's just hope it's a commit
712
if type(tag_name) is not unicode:
713
raise TypeError(tag_name)
714
yield (ref_name, tag_name, peeled, unpeeled)
717
def remote_refs_dict_to_container(refs_dict, symrefs_dict={}):
720
for k, v in refs_dict.iteritems():
726
for name, target in symrefs_dict.iteritems():
727
base[name] = SYMREF + target
728
ret = DictRefsContainer(base)
added
removed
remote.py
