1
# Copyright (C) 2007-2018 Jelmer Vernooij <jelmer@jelmer.uk>
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Remote dirs, repositories and branches."""
19
from __future__ import absolute_import
32
from ...errors import (
44
UninitializableFormat,
46
from ...transport import (
65
BareLocalGitControlDirFormat,
68
GitSmartRemoteNotSupported,
71
from .mapping import (
74
from .object_store import (
80
from .repository import (
92
from dulwich.errors import (
95
from dulwich.pack import (
99
from dulwich.protocol import ZERO_SHA
100
from dulwich.refs import SYMREF
101
from dulwich.repo import DictRefsContainer
108
# urlparse only supports a limited number of schemes by default
110
urlparse.uses_netloc.extend(['git', 'git+ssh'])
112
from dulwich.pack import load_pack_index
115
class GitPushResult(PushResult):
117
def _lookup_revno(self, revid):
119
return _quick_lookup_revno(self.source_branch, self.target_branch,
121
except GitSmartRemoteNotSupported:
126
return self._lookup_revno(self.old_revid)
130
return self._lookup_revno(self.new_revid)
133
# Don't run any tests on GitSmartTransport as it is not intended to be
134
# a full implementation of Transport
135
def get_test_permutations():
139
def split_git_url(url):
143
:return: Tuple with host, port, username, path.
145
(scheme, netloc, loc, _, _) = urlparse.urlsplit(url)
146
path = urllib.unquote(loc)
147
if path.startswith("/~"):
149
(username, hostport) = urllib.splituser(netloc)
150
(host, port) = urllib.splitnport(hostport, None)
151
return (host, port, username, path)
154
class RemoteGitError(BzrError):
156
_fmt = "Remote server error: %(msg)s"
159
def parse_git_error(url, message):
160
"""Parse a remote git server error and return a bzr exception.
162
:param url: URL of the remote repository
163
:param message: Message sent by the remote git server
165
message = str(message).strip()
166
if message.startswith("Could not find Repository "):
167
return NotBranchError(url, message)
168
if message == "HEAD failed to update":
169
base_url, _ = urlutils.split_segment_parameters(url)
171
("Unable to update remote HEAD branch. To update the master "
172
"branch, specify the URL %s,branch=master.") % base_url)
173
# Don't know, just return it to the user as-is
174
return RemoteGitError(message)
177
class GitSmartTransport(Transport):
179
def __init__(self, url, _client=None):
180
Transport.__init__(self, url)
181
(self._host, self._port, self._username, self._path) = \
183
if 'transport' in debug.debug_flags:
184
trace.mutter('host: %r, user: %r, port: %r, path: %r',
185
self._host, self._username, self._port, self._path)
186
self._client = _client
187
self._stripped_path = self._path.rsplit(",", 1)[0]
189
def external_url(self):
192
def has(self, relpath):
195
def _get_client(self):
196
raise NotImplementedError(self._get_client)
199
return self._stripped_path
202
raise NoSuchFile(path)
204
def abspath(self, relpath):
205
return urlutils.join(self.base, relpath)
207
def clone(self, offset=None):
208
"""See Transport.clone()."""
212
newurl = urlutils.join(self.base, offset)
214
return self.__class__(newurl, self._client)
217
class TCPGitSmartTransport(GitSmartTransport):
221
def _get_client(self):
222
if self._client is not None:
227
# return dulwich.client.LocalGitClient()
228
return dulwich.client.SubprocessGitClient()
229
return dulwich.client.TCPGitClient(self._host, self._port,
230
report_activity=self._report_activity)
233
class SSHSocketWrapper(object):
235
def __init__(self, sock):
238
def read(self, len=None):
239
return self.sock.recv(len)
241
def write(self, data):
242
return self.sock.write(data)
245
return len(select.select([self.sock.fileno()], [], [], 0)[0]) > 0
248
class DulwichSSHVendor(dulwich.client.SSHVendor):
251
from ...transport import ssh
252
self.bzr_ssh_vendor = ssh._get_ssh_vendor()
254
def run_command(self, host, command, username=None, port=None):
255
connection = self.bzr_ssh_vendor.connect_ssh(username=username,
256
password=None, port=port, host=host, command=command)
257
(kind, io_object) = connection.get_sock_or_pipes()
259
return SSHSocketWrapper(io_object)
261
raise AssertionError("Unknown io object kind %r'" % kind)
264
#dulwich.client.get_ssh_vendor = DulwichSSHVendor
267
class SSHGitSmartTransport(GitSmartTransport):
272
path = self._stripped_path
273
if path.startswith("/~/"):
277
def _get_client(self):
278
if self._client is not None:
282
location_config = config.LocationConfig(self.base)
283
client = dulwich.client.SSHGitClient(self._host, self._port, self._username,
284
report_activity=self._report_activity)
285
# Set up alternate pack program paths
286
upload_pack = location_config.get_user_option('git_upload_pack')
288
client.alternative_paths["upload-pack"] = upload_pack
289
receive_pack = location_config.get_user_option('git_receive_pack')
291
client.alternative_paths["receive-pack"] = receive_pack
295
class RemoteGitBranchFormat(GitBranchFormat):
297
def get_format_description(self):
298
return 'Remote Git Branch'
301
def _matchingcontroldir(self):
302
return RemoteGitControlDirFormat()
304
def initialize(self, a_controldir, name=None, repository=None,
305
append_revisions_only=None):
306
raise UninitializableFormat(self)
309
def default_report_progress(text):
310
if text.startswith('error: '):
311
trace.show_error('git: %s', text[len('error: '):])
313
trace.mutter("git: %s" % text)
316
class RemoteGitDir(GitDir):
318
def __init__(self, transport, format, client, client_path):
319
self._format = format
320
self.root_transport = transport
321
self.transport = transport
322
self._mode_check_done = None
323
self._client = client
324
self._client_path = client_path
325
self.base = self.root_transport.base
329
def _gitrepository_class(self):
330
return RemoteGitRepository
332
def fetch_pack(self, determine_wants, graph_walker, pack_data, progress=None):
334
progress = default_report_progress
336
result = self._client.fetch_pack(self._client_path, determine_wants,
337
graph_walker, pack_data, progress)
338
if result.refs is None:
340
self._refs = remote_refs_dict_to_container(result.refs, result.symrefs)
342
except GitProtocolError, e:
343
raise parse_git_error(self.transport.external_url(), e)
345
def send_pack(self, get_changed_refs, generate_pack_data, progress=None):
347
progress = default_report_progress
350
return self._client.send_pack(self._client_path, get_changed_refs,
351
generate_pack_data, progress)
352
except GitProtocolError, e:
353
raise parse_git_error(self.transport.external_url(), e)
355
def create_branch(self, name=None, repository=None,
356
append_revisions_only=None, ref=None):
357
refname = self._get_selected_ref(name, ref)
358
if refname != b'HEAD' and refname in self.get_refs_container():
359
raise AlreadyBranchError(self.user_url)
360
if refname in self.get_refs_container():
361
ref_chain, unused_sha = self.get_refs_container().follow(self._get_selected_ref(None))
362
if ref_chain[0] == b'HEAD':
363
refname = ref_chain[1]
364
repo = self.open_repository()
365
return RemoteGitBranch(self, repo, refname)
367
def destroy_branch(self, name=None):
368
refname = self._get_selected_ref(name)
369
def get_changed_refs(old_refs):
371
if not refname in ret:
372
raise NotBranchError(self.user_url)
373
ret[refname] = dulwich.client.ZERO_SHA
375
def generate_pack_data(have, want, ofs_delta=False):
376
return pack_objects_to_data([])
377
self.send_pack(get_changed_refs, generate_pack_data)
381
return self.control_url
384
def user_transport(self):
385
return self.root_transport
388
def control_url(self):
389
return self.control_transport.base
392
def control_transport(self):
393
return self.root_transport
395
def open_repository(self):
396
return RemoteGitRepository(self)
398
def open_branch(self, name=None, unsupported=False,
399
ignore_fallbacks=False, ref=None, possible_transports=None,
401
repo = self.open_repository()
402
ref = self._get_selected_ref(name, ref)
403
if not nascent_ok and ref not in self.get_refs_container():
404
raise NotBranchError(self.root_transport.base,
406
ref_chain, unused_sha = self.get_refs_container().follow(ref)
407
return RemoteGitBranch(self, repo, ref_chain[-1])
409
def open_workingtree(self, recommend_upgrade=False):
410
raise NotLocalUrl(self.transport.base)
412
def has_workingtree(self):
415
def get_peeled(self, name):
416
return self.get_refs_container().get_peeled(name)
418
def get_refs_container(self):
419
if self._refs is not None:
421
result = self.fetch_pack(lambda x: None, None,
422
lambda x: None, lambda x: trace.mutter("git: %s" % x))
423
self._refs = remote_refs_dict_to_container(
424
result.refs, result.symrefs)
427
def push_branch(self, source, revision_id=None, overwrite=False,
428
remember=False, create_prefix=False, lossy=False,
430
"""Push the source branch into this ControlDir."""
431
if revision_id is None:
432
# No revision supplied by the user, default to the branch
434
revision_id = source.last_revision()
436
push_result = GitPushResult()
437
push_result.workingtree_updated = None
438
push_result.master_branch = None
439
push_result.source_branch = source
440
push_result.stacked_on = None
441
push_result.branch_push_result = None
442
repo = self.find_repository()
443
refname = self._get_selected_ref(name)
444
source_store = get_object_store(source.repository)
445
with source_store.lock_read():
446
def get_changed_refs(refs):
447
self._refs = remote_refs_dict_to_container(refs)
449
# TODO(jelmer): Unpeel if necessary
451
new_sha = source_store._lookup_revision_sha1(revision_id)
453
new_sha = repo.lookup_bzr_revision_id(revision_id)[0]
455
if remote_divergence(ret.get(refname), new_sha, source_store):
456
raise DivergedBranches(
457
source, self.open_branch(name, nascent_ok=True))
458
ret[refname] = new_sha
461
generate_pack_data = source_store.generate_lossy_pack_data
463
generate_pack_data = source_store.generate_pack_data
464
new_refs = self.send_pack(get_changed_refs, generate_pack_data)
465
push_result.new_revid = repo.lookup_foreign_revision_id(
468
old_remote = self._refs[refname]
470
old_remote = ZERO_SHA
471
push_result.old_revid = repo.lookup_foreign_revision_id(old_remote)
472
self._refs = remote_refs_dict_to_container(new_refs)
473
push_result.target_branch = self.open_branch(name)
474
if old_remote != ZERO_SHA:
475
push_result.branch_push_result = GitBranchPushResult()
476
push_result.branch_push_result.source_branch = source
477
push_result.branch_push_result.target_branch = push_result.target_branch
478
push_result.branch_push_result.local_branch = None
479
push_result.branch_push_result.master_branch = push_result.target_branch
480
push_result.branch_push_result.old_revid = push_result.old_revid
481
push_result.branch_push_result.new_revid = push_result.new_revid
482
if source.get_push_location() is None or remember:
483
source.set_push_location(push_result.target_branch.base)
487
class EmptyObjectStoreIterator(dict):
489
def iterobjects(self):
493
class TemporaryPackIterator(Pack):
495
def __init__(self, path, resolve_ext_ref):
496
super(TemporaryPackIterator, self).__init__(
497
path, resolve_ext_ref=resolve_ext_ref)
498
self._idx_load = lambda: self._idx_load_or_generate(self._idx_path)
500
def _idx_load_or_generate(self, path):
501
if not os.path.exists(path):
502
pb = ui.ui_factory.nested_progress_bar()
504
def report_progress(cur, total):
505
pb.update("generating index", cur, total)
506
self.data.create_index(path,
507
progress=report_progress)
510
return load_pack_index(path)
513
if self._idx is not None:
515
os.remove(self._idx_path)
516
if self._data is not None:
518
os.remove(self._data_path)
521
class BzrGitHttpClient(dulwich.client.HttpGitClient):
523
def __init__(self, transport, *args, **kwargs):
524
self.transport = transport
525
super(BzrGitHttpClient, self).__init__(transport.external_url(), *args, **kwargs)
527
self._http_perform = getattr(self.transport, "_perform", urllib2.urlopen)
529
def _perform(self, req):
530
req.accepted_errors = (200, 404)
531
req.follow_redirections = True
532
req.redirected_to = None
533
return self._http_perform(req)
536
class RemoteGitControlDirFormat(GitControlDirFormat):
537
"""The .git directory control format."""
539
supports_workingtrees = False
542
def _known_formats(self):
543
return set([RemoteGitControlDirFormat()])
545
def get_branch_format(self):
546
return RemoteGitBranchFormat()
548
def is_initializable(self):
551
def is_supported(self):
554
def open(self, transport, _found=None):
555
"""Open this directory.
558
# we dont grok readonly - git isn't integrated with transport.
560
if url.startswith('readonly+'):
561
url = url[len('readonly+'):]
562
scheme = urlparse.urlsplit(transport.external_url())[0]
563
if isinstance(transport, GitSmartTransport):
564
client = transport._get_client()
565
client_path = transport._get_path()
566
elif scheme in ("http", "https"):
567
client = BzrGitHttpClient(transport)
568
client_path, _ = urlutils.split_segment_parameters(transport._path)
569
elif scheme == 'file':
570
client = dulwich.client.LocalGitClient()
571
client_path = transport.local_abspath('.')
573
raise NotBranchError(transport.base)
575
pass # TODO(jelmer): Actually probe for something
576
return RemoteGitDir(transport, self, client, client_path)
578
def get_format_description(self):
579
return "Remote Git Repository"
581
def initialize_on_transport(self, transport):
582
raise UninitializableFormat(self)
584
def supports_transport(self, transport):
586
external_url = transport.external_url()
587
except InProcessTransport:
588
raise NotBranchError(path=transport.base)
589
return (external_url.startswith("http:") or
590
external_url.startswith("https:") or
591
external_url.startswith("git+") or
592
external_url.startswith("git:"))
595
class RemoteGitRepository(GitRepository):
599
return self.control_url
601
def get_parent_map(self, revids):
602
raise GitSmartRemoteNotSupported(self.get_parent_map, self)
604
def fetch_pack(self, determine_wants, graph_walker, pack_data,
606
return self.controldir.fetch_pack(determine_wants, graph_walker,
609
def send_pack(self, get_changed_refs, generate_pack_data):
610
return self.controldir.send_pack(get_changed_refs, generate_pack_data)
612
def fetch_objects(self, determine_wants, graph_walker, resolve_ext_ref,
614
fd, path = tempfile.mkstemp(suffix=".pack")
616
self.fetch_pack(determine_wants, graph_walker,
617
lambda x: os.write(fd, x), progress)
620
if os.path.getsize(path) == 0:
621
return EmptyObjectStoreIterator()
622
return TemporaryPackIterator(path[:-len(".pack")], resolve_ext_ref)
624
def lookup_bzr_revision_id(self, bzr_revid, mapping=None):
625
# This won't work for any round-tripped bzr revisions, but it's a start..
627
return mapping_registry.revision_id_bzr_to_foreign(bzr_revid)
628
except InvalidRevisionId:
629
raise NoSuchRevision(self, bzr_revid)
631
def lookup_foreign_revision_id(self, foreign_revid, mapping=None):
632
"""Lookup a revision id.
636
mapping = self.get_mapping()
637
# Not really an easy way to parse foreign revids here..
638
return mapping.revision_id_foreign_to_bzr(foreign_revid)
640
def revision_tree(self, revid):
641
raise GitSmartRemoteNotSupported(self.revision_tree, self)
643
def get_revisions(self, revids):
644
raise GitSmartRemoteNotSupported(self.get_revisions, self)
646
def has_revisions(self, revids):
647
raise GitSmartRemoteNotSupported(self.get_revisions, self)
650
class RemoteGitTagDict(GitTags):
652
def set_tag(self, name, revid):
653
sha = self.branch.lookup_bzr_revision_id(revid)[0]
654
self._set_ref(name, sha)
656
def delete_tag(self, name):
657
self._set_ref(name, dulwich.client.ZERO_SHA)
659
def _set_ref(self, name, sha):
660
ref = tag_name_to_ref(name)
661
def get_changed_refs(old_refs):
663
if sha == dulwich.client.ZERO_SHA and ref not in ret:
664
raise NoSuchTag(name)
667
def generate_pack_data(have, want, ofs_delta=False):
668
return pack_objects_to_data([])
669
self.repository.send_pack(get_changed_refs, generate_pack_data)
672
class RemoteGitBranch(GitBranch):
674
def __init__(self, controldir, repository, name):
676
super(RemoteGitBranch, self).__init__(controldir, repository, name,
677
RemoteGitBranchFormat())
679
def last_revision_info(self):
680
raise GitSmartRemoteNotSupported(self.last_revision_info, self)
684
return self.control_url
687
def control_url(self):
690
def revision_id_to_revno(self, revision_id):
691
raise GitSmartRemoteNotSupported(self.revision_id_to_revno, self)
693
def last_revision(self):
694
return self.lookup_foreign_revision_id(self.head)
698
if self._sha is not None:
700
refs = self.controldir.get_refs_container()
701
name = branch_name_to_ref(self.name)
703
self._sha = refs[name]
705
raise NoSuchRef(name, self.repository.user_url, refs)
708
def _synchronize_history(self, destination, revision_id):
709
"""See Branch._synchronize_history()."""
710
destination.generate_revision_history(self.last_revision())
712
def _get_parent_location(self):
715
def get_push_location(self):
718
def set_push_location(self, url):
721
def _iter_tag_refs(self):
722
"""Iterate over the tag refs.
724
:param refs: Refs dictionary (name -> git sha1)
725
:return: iterator over (ref_name, tag_name, peeled_sha1, unpeeled_sha1)
727
refs = self.controldir.get_refs_container()
728
for ref_name, unpeeled in refs.as_dict().iteritems():
730
tag_name = ref_to_tag_name(ref_name)
731
except (ValueError, UnicodeDecodeError):
733
peeled = refs.get_peeled(ref_name)
736
peeled = refs.peel_sha(unpeeled).id
738
# Let's just hope it's a commit
740
if type(tag_name) is not unicode:
741
raise TypeError(tag_name)
742
yield (ref_name, tag_name, peeled, unpeeled)
745
def remote_refs_dict_to_container(refs_dict, symrefs_dict={}):
748
for k, v in refs_dict.iteritems():
754
for name, target in symrefs_dict.iteritems():
755
base[name] = SYMREF + target
756
ret = DictRefsContainer(base)
added
removed
remote.py
