1
# Copyright (C) 2007-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Remote dirs, repositories and branches."""
19
from __future__ import absolute_import
34
from ...errors import (
46
UninitializableFormat,
48
from ...transport import (
66
BareLocalGitControlDirFormat,
69
GitSmartRemoteNotSupported,
72
from .mapping import (
75
from .object_store import (
81
from .repository import (
93
from dulwich.errors import (
96
from dulwich.pack import (
100
from dulwich.protocol import ZERO_SHA
101
from dulwich.refs import SYMREF
102
from dulwich.repo import DictRefsContainer
109
# urlparse only supports a limited number of schemes by default
111
urlparse.uses_netloc.extend(['git', 'git+ssh'])
113
from dulwich.pack import load_pack_index
116
# Don't run any tests on GitSmartTransport as it is not intended to be
117
# a full implementation of Transport
118
def get_test_permutations():
122
def split_git_url(url):
126
:return: Tuple with host, port, username, path.
128
(scheme, netloc, loc, _, _) = urlparse.urlsplit(url)
129
path = urllib.unquote(loc)
130
if path.startswith("/~"):
132
(username, hostport) = urllib.splituser(netloc)
133
(host, port) = urllib.splitnport(hostport, None)
134
return (host, port, username, path)
137
class RemoteGitError(BzrError):
139
_fmt = "Remote server error: %(msg)s"
142
def parse_git_error(url, message):
143
"""Parse a remote git server error and return a bzr exception.
145
:param url: URL of the remote repository
146
:param message: Message sent by the remote git server
148
message = str(message).strip()
149
if message.startswith("Could not find Repository "):
150
return NotBranchError(url, message)
151
if message == "HEAD failed to update":
152
base_url, _ = urlutils.split_segment_parameters(url)
154
("Unable to update remote HEAD branch. To update the master "
155
"branch, specify the URL %s,branch=master.") % base_url)
156
# Don't know, just return it to the user as-is
157
return RemoteGitError(message)
160
class GitSmartTransport(Transport):
162
def __init__(self, url, _client=None):
163
Transport.__init__(self, url)
164
(self._host, self._port, self._username, self._path) = \
166
if 'transport' in debug.debug_flags:
167
trace.mutter('host: %r, user: %r, port: %r, path: %r',
168
self._host, self._username, self._port, self._path)
169
self._client = _client
170
self._stripped_path = self._path.rsplit(",", 1)[0]
172
def external_url(self):
175
def has(self, relpath):
178
def _get_client(self):
179
raise NotImplementedError(self._get_client)
182
return self._stripped_path
185
raise NoSuchFile(path)
187
def abspath(self, relpath):
188
return urlutils.join(self.base, relpath)
190
def clone(self, offset=None):
191
"""See Transport.clone()."""
195
newurl = urlutils.join(self.base, offset)
197
return self.__class__(newurl, self._client)
200
class TCPGitSmartTransport(GitSmartTransport):
204
def _get_client(self):
205
if self._client is not None:
210
# return dulwich.client.LocalGitClient()
211
return dulwich.client.SubprocessGitClient()
212
return dulwich.client.TCPGitClient(self._host, self._port,
213
report_activity=self._report_activity)
216
class SSHSocketWrapper(object):
218
def __init__(self, sock):
221
def read(self, len=None):
222
return self.sock.recv(len)
224
def write(self, data):
225
return self.sock.write(data)
228
return len(select.select([self.sock.fileno()], [], [], 0)[0]) > 0
231
class DulwichSSHVendor(dulwich.client.SSHVendor):
234
from ...transport import ssh
235
self.bzr_ssh_vendor = ssh._get_ssh_vendor()
237
def run_command(self, host, command, username=None, port=None):
238
connection = self.bzr_ssh_vendor.connect_ssh(username=username,
239
password=None, port=port, host=host, command=command)
240
(kind, io_object) = connection.get_sock_or_pipes()
242
return SSHSocketWrapper(io_object)
244
raise AssertionError("Unknown io object kind %r'" % kind)
247
#dulwich.client.get_ssh_vendor = DulwichSSHVendor
250
class SSHGitSmartTransport(GitSmartTransport):
255
path = self._stripped_path
256
if path.startswith("/~/"):
260
def _get_client(self):
261
if self._client is not None:
265
location_config = config.LocationConfig(self.base)
266
client = dulwich.client.SSHGitClient(self._host, self._port, self._username,
267
report_activity=self._report_activity)
268
# Set up alternate pack program paths
269
upload_pack = location_config.get_user_option('git_upload_pack')
271
client.alternative_paths["upload-pack"] = upload_pack
272
receive_pack = location_config.get_user_option('git_receive_pack')
274
client.alternative_paths["receive-pack"] = receive_pack
278
class RemoteGitBranchFormat(GitBranchFormat):
280
def get_format_description(self):
281
return 'Remote Git Branch'
284
def _matchingcontroldir(self):
285
return RemoteGitControlDirFormat()
287
def initialize(self, a_controldir, name=None, repository=None,
288
append_revisions_only=None):
289
raise UninitializableFormat(self)
292
_GIT_PROGRESS_PARTIAL_RE = re.compile(r"(.*?): +(\d+)% \((\d+)/(\d+)\)")
293
_GIT_PROGRESS_TOTAL_RE = re.compile(r"(.*?): (\d+)")
294
def default_report_progress(pb, text):
295
text = text.rstrip("\r\n")
296
if text.startswith('error: '):
297
trace.show_error('git: %s', text[len('error: '):])
299
trace.mutter("git: %s", text)
300
g = _GIT_PROGRESS_PARTIAL_RE.match(text)
302
(text, pct, current, total) = g.groups()
303
pb.update(text, int(current), int(total))
305
g = _GIT_PROGRESS_TOTAL_RE.match(text)
307
(text, total) = g.groups()
308
pb.update(text, None, int(total))
310
pb.update(text, None, None)
313
class RemoteGitDir(GitDir):
315
def __init__(self, transport, format, client, client_path):
316
self._format = format
317
self.root_transport = transport
318
self.transport = transport
319
self._mode_check_done = None
320
self._client = client
321
self._client_path = client_path
322
self.base = self.root_transport.base
326
def _gitrepository_class(self):
327
return RemoteGitRepository
329
def fetch_pack(self, determine_wants, graph_walker, pack_data, progress=None):
331
progress = lambda text: default_report_progress(pb, text)
332
pb = ui.ui_factory.nested_progress_bar()
336
result = self._client.fetch_pack(self._client_path, determine_wants,
337
graph_walker, pack_data, progress)
338
if result.refs is None:
340
self._refs = remote_refs_dict_to_container(result.refs, result.symrefs)
342
except GitProtocolError, e:
343
raise parse_git_error(self.transport.external_url(), e)
348
def send_pack(self, get_changed_refs, generate_pack_data, progress=None):
350
progress = lambda text: default_report_progress(pb, text)
351
pb = ui.ui_factory.nested_progress_bar()
355
return self._client.send_pack(self._client_path, get_changed_refs,
356
generate_pack_data, progress)
357
except GitProtocolError, e:
358
raise parse_git_error(self.transport.external_url(), e)
363
def create_branch(self, name=None, repository=None,
364
append_revisions_only=None, ref=None):
365
refname = self._get_selected_ref(name, ref)
366
if refname != b'HEAD' and refname in self.get_refs_container():
367
raise AlreadyBranchError(self.user_url)
368
if refname in self.get_refs_container():
369
ref_chain, unused_sha = self.get_refs_container().follow(self._get_selected_ref(None))
370
if ref_chain[0] == b'HEAD':
371
refname = ref_chain[1]
372
repo = self.open_repository()
373
return RemoteGitBranch(self, repo, refname)
375
def destroy_branch(self, name=None):
376
refname = self._get_selected_ref(name)
377
def get_changed_refs(old_refs):
379
if not refname in ret:
380
raise NotBranchError(self.user_url)
381
ret[refname] = dulwich.client.ZERO_SHA
383
def generate_pack_data(have, want, ofs_delta=False):
384
return pack_objects_to_data([])
385
self.send_pack(get_changed_refs, generate_pack_data)
389
return self.control_url
392
def user_transport(self):
393
return self.root_transport
396
def control_url(self):
397
return self.control_transport.base
400
def control_transport(self):
401
return self.root_transport
403
def open_repository(self):
404
return RemoteGitRepository(self)
406
def open_branch(self, name=None, unsupported=False,
407
ignore_fallbacks=False, ref=None, possible_transports=None,
409
repo = self.open_repository()
410
ref = self._get_selected_ref(name, ref)
411
if not nascent_ok and ref not in self.get_refs_container():
412
raise NotBranchError(self.root_transport.base,
414
ref_chain, unused_sha = self.get_refs_container().follow(ref)
415
return RemoteGitBranch(self, repo, ref_chain[-1])
417
def open_workingtree(self, recommend_upgrade=False):
418
raise NotLocalUrl(self.transport.base)
420
def has_workingtree(self):
423
def get_peeled(self, name):
424
return self.get_refs_container().get_peeled(name)
426
def get_refs_container(self):
427
if self._refs is not None:
429
result = self.fetch_pack(lambda x: None, None,
430
lambda x: None, lambda x: trace.mutter("git: %s" % x))
431
self._refs = remote_refs_dict_to_container(
432
result.refs, result.symrefs)
435
def push_branch(self, source, revision_id=None, overwrite=False,
436
remember=False, create_prefix=False, lossy=False,
438
"""Push the source branch into this ControlDir."""
439
if revision_id is None:
440
# No revision supplied by the user, default to the branch
442
revision_id = source.last_revision()
444
push_result = PushResult()
445
push_result.workingtree_updated = None
446
push_result.master_branch = None
447
push_result.source_branch = source
448
push_result.stacked_on = None
449
push_result.branch_push_result = None
450
repo = self.find_repository()
451
refname = self._get_selected_ref(name)
452
source_store = get_object_store(source.repository)
453
with source_store.lock_read():
454
def get_changed_refs(refs):
455
self._refs = remote_refs_dict_to_container(refs)
457
# TODO(jelmer): Unpeel if necessary
459
new_sha = source_store._lookup_revision_sha1(revision_id)
461
new_sha = repo.lookup_bzr_revision_id(revision_id)[0]
463
if remote_divergence(ret.get(refname), new_sha, source_store):
464
raise DivergedBranches(
465
source, self.open_branch(name, nascent_ok=True))
466
ret[refname] = new_sha
469
generate_pack_data = source_store.generate_lossy_pack_data
471
generate_pack_data = source_store.generate_pack_data
472
new_refs = self.send_pack(get_changed_refs, generate_pack_data)
473
push_result.new_revid = repo.lookup_foreign_revision_id(
476
old_remote = self._refs[refname]
478
old_remote = ZERO_SHA
479
push_result.old_revid = repo.lookup_foreign_revision_id(old_remote)
480
self._refs = remote_refs_dict_to_container(new_refs)
481
push_result.old_revno = None
482
push_result.target_branch = self.open_branch(name)
483
if old_remote != ZERO_SHA:
484
push_result.branch_push_result = GitBranchPushResult()
485
push_result.branch_push_result.source_branch = source
486
push_result.branch_push_result.target_branch = push_result.target_branch
487
push_result.branch_push_result.local_branch = None
488
push_result.branch_push_result.master_branch = push_result.target_branch
489
push_result.branch_push_result.old_revid = push_result.old_revid
490
push_result.branch_push_result.new_revid = push_result.new_revid
491
if source.get_push_location() is None or remember:
492
source.set_push_location(push_result.target_branch.base)
496
class EmptyObjectStoreIterator(dict):
498
def iterobjects(self):
502
class TemporaryPackIterator(Pack):
504
def __init__(self, path, resolve_ext_ref):
505
super(TemporaryPackIterator, self).__init__(
506
path, resolve_ext_ref=resolve_ext_ref)
507
self._idx_load = lambda: self._idx_load_or_generate(self._idx_path)
509
def _idx_load_or_generate(self, path):
510
if not os.path.exists(path):
511
pb = ui.ui_factory.nested_progress_bar()
513
def report_progress(cur, total):
514
pb.update("generating index", cur, total)
515
self.data.create_index(path,
516
progress=report_progress)
519
return load_pack_index(path)
522
if self._idx is not None:
524
os.remove(self._idx_path)
525
if self._data is not None:
527
os.remove(self._data_path)
530
class BzrGitHttpClient(dulwich.client.HttpGitClient):
532
def __init__(self, transport, *args, **kwargs):
533
self.transport = transport
534
super(BzrGitHttpClient, self).__init__(transport.external_url(), *args, **kwargs)
536
self._http_perform = getattr(self.transport, "_perform", urllib2.urlopen)
538
def _perform(self, req):
539
req.accepted_errors = (200, 404)
540
req.follow_redirections = True
541
req.redirected_to = None
542
return self._http_perform(req)
545
class RemoteGitControlDirFormat(GitControlDirFormat):
546
"""The .git directory control format."""
548
supports_workingtrees = False
551
def _known_formats(self):
552
return set([RemoteGitControlDirFormat()])
554
def get_branch_format(self):
555
return RemoteGitBranchFormat()
557
def is_initializable(self):
560
def is_supported(self):
563
def open(self, transport, _found=None):
564
"""Open this directory.
567
# we dont grok readonly - git isn't integrated with transport.
569
if url.startswith('readonly+'):
570
url = url[len('readonly+'):]
571
scheme = urlparse.urlsplit(transport.external_url())[0]
572
if isinstance(transport, GitSmartTransport):
573
client = transport._get_client()
574
client_path = transport._get_path()
575
elif scheme in ("http", "https"):
576
client = BzrGitHttpClient(transport)
577
client_path, _ = urlutils.split_segment_parameters(transport._path)
578
elif scheme == 'file':
579
client = dulwich.client.LocalGitClient()
580
client_path = transport.local_abspath('.')
582
raise NotBranchError(transport.base)
584
pass # TODO(jelmer): Actually probe for something
585
return RemoteGitDir(transport, self, client, client_path)
587
def get_format_description(self):
588
return "Remote Git Repository"
590
def initialize_on_transport(self, transport):
591
raise UninitializableFormat(self)
593
def supports_transport(self, transport):
595
external_url = transport.external_url()
596
except InProcessTransport:
597
raise NotBranchError(path=transport.base)
598
return (external_url.startswith("http:") or
599
external_url.startswith("https:") or
600
external_url.startswith("git+") or
601
external_url.startswith("git:"))
604
class RemoteGitRepository(GitRepository):
608
return self.control_url
610
def get_parent_map(self, revids):
611
raise GitSmartRemoteNotSupported(self.get_parent_map, self)
613
def fetch_pack(self, determine_wants, graph_walker, pack_data,
615
return self.controldir.fetch_pack(determine_wants, graph_walker,
618
def send_pack(self, get_changed_refs, generate_pack_data):
619
return self.controldir.send_pack(get_changed_refs, generate_pack_data)
621
def fetch_objects(self, determine_wants, graph_walker, resolve_ext_ref,
623
fd, path = tempfile.mkstemp(suffix=".pack")
625
self.fetch_pack(determine_wants, graph_walker,
626
lambda x: os.write(fd, x), progress)
629
if os.path.getsize(path) == 0:
630
return EmptyObjectStoreIterator()
631
return TemporaryPackIterator(path[:-len(".pack")], resolve_ext_ref)
633
def lookup_bzr_revision_id(self, bzr_revid, mapping=None):
634
# This won't work for any round-tripped bzr revisions, but it's a start..
636
return mapping_registry.revision_id_bzr_to_foreign(bzr_revid)
637
except InvalidRevisionId:
638
raise NoSuchRevision(self, bzr_revid)
640
def lookup_foreign_revision_id(self, foreign_revid, mapping=None):
641
"""Lookup a revision id.
645
mapping = self.get_mapping()
646
# Not really an easy way to parse foreign revids here..
647
return mapping.revision_id_foreign_to_bzr(foreign_revid)
649
def revision_tree(self, revid):
650
raise GitSmartRemoteNotSupported(self.revision_tree, self)
652
def get_revisions(self, revids):
653
raise GitSmartRemoteNotSupported(self.get_revisions, self)
655
def has_revisions(self, revids):
656
raise GitSmartRemoteNotSupported(self.get_revisions, self)
659
class RemoteGitTagDict(GitTags):
661
def set_tag(self, name, revid):
662
sha = self.branch.lookup_bzr_revision_id(revid)[0]
663
self._set_ref(name, sha)
665
def delete_tag(self, name):
666
self._set_ref(name, dulwich.client.ZERO_SHA)
668
def _set_ref(self, name, sha):
669
ref = tag_name_to_ref(name)
670
def get_changed_refs(old_refs):
672
if sha == dulwich.client.ZERO_SHA and ref not in ret:
673
raise NoSuchTag(name)
676
def generate_pack_data(have, want, ofs_delta=False):
677
return pack_objects_to_data([])
678
self.repository.send_pack(get_changed_refs, generate_pack_data)
681
class RemoteGitBranch(GitBranch):
683
def __init__(self, controldir, repository, name):
685
super(RemoteGitBranch, self).__init__(controldir, repository, name,
686
RemoteGitBranchFormat())
688
def last_revision_info(self):
689
raise GitSmartRemoteNotSupported(self.last_revision_info, self)
693
return self.control_url
696
def control_url(self):
699
def revision_id_to_revno(self, revision_id):
700
raise GitSmartRemoteNotSupported(self.revision_id_to_revno, self)
702
def last_revision(self):
703
return self.lookup_foreign_revision_id(self.head)
707
if self._sha is not None:
709
refs = self.controldir.get_refs_container()
710
name = branch_name_to_ref(self.name)
712
self._sha = refs[name]
714
raise NoSuchRef(name, self.repository.user_url, refs)
717
def _synchronize_history(self, destination, revision_id):
718
"""See Branch._synchronize_history()."""
719
destination.generate_revision_history(self.last_revision())
721
def _get_parent_location(self):
724
def get_push_location(self):
727
def set_push_location(self, url):
730
def _iter_tag_refs(self):
731
"""Iterate over the tag refs.
733
:param refs: Refs dictionary (name -> git sha1)
734
:return: iterator over (ref_name, tag_name, peeled_sha1, unpeeled_sha1)
736
refs = self.controldir.get_refs_container()
737
for ref_name, unpeeled in refs.as_dict().iteritems():
739
tag_name = ref_to_tag_name(ref_name)
740
except (ValueError, UnicodeDecodeError):
742
peeled = refs.get_peeled(ref_name)
745
peeled = refs.peel_sha(unpeeled).id
747
# Let's just hope it's a commit
749
if type(tag_name) is not unicode:
750
raise TypeError(tag_name)
751
yield (ref_name, tag_name, peeled, unpeeled)
754
def remote_refs_dict_to_container(refs_dict, symrefs_dict={}):
757
for k, v in refs_dict.iteritems():
763
for name, target in symrefs_dict.iteritems():
764
base[name] = SYMREF + target
765
ret = DictRefsContainer(base)
added
removed
remote.py
