/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/git/remote.py

  • Committer: Jelmer Vernooij
  • Date: 2019-05-29 03:22:34 UTC
  • mfrom: (7303 work)
  • mto: This revision was merged to the branch mainline in revision 7306.
  • Revision ID: jelmer@jelmer.uk-20190529032234-mt3fuws8gq03tapi
Merge trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/git/remote.py
18
18
 
19
19
from __future__ import absolute_import
20
20
 
21
 
from io import BytesIO
 
21
import gzip
22
22
import re
23
23
 
24
24
from .. import (
44
44
    NoSuchTag,
45
45
    NotBranchError,
46
46
    NotLocalUrl,
47
 
    NoWorkingTree,
48
47
    PermissionDenied,
49
48
    UninitializableFormat,
50
49
    )
51
50
from ..revisiontree import RevisionTree
52
 
from ..sixish import text_type
 
51
from ..sixish import (
 
52
    text_type,
 
53
    viewitems,
 
54
    )
53
55
from ..transport import (
54
56
    Transport,
55
57
    register_urlparse_netloc_protocol,
71
73
from .dir import (
72
74
    GitControlDirFormat,
73
75
    GitDir,
74
 
    BareLocalGitControlDirFormat,
75
76
    )
76
77
from .errors import (
77
78
    GitSmartRemoteNotSupported,
100
101
import dulwich.client
101
102
from dulwich.errors import (
102
103
    GitProtocolError,
 
104
    HangupException,
103
105
    )
104
106
from dulwich.pack import (
105
107
    Pack,
116
118
import os
117
119
import select
118
120
import tempfile
119
 
import urllib
120
121
 
121
122
try:
122
123
    import urllib.parse as urlparse
137
138
    def _lookup_revno(self, revid):
138
139
        try:
139
140
            return _quick_lookup_revno(self.source_branch, self.target_branch,
140
 
                revid)
 
141
                                       revid)
141
142
        except GitSmartRemoteNotSupported:
142
143
            return None
143
144
 
193
194
    :param message: Message sent by the remote git server
194
195
    """
195
196
    message = str(message).strip()
196
 
    if (message.startswith("Could not find Repository ") or
197
 
        message == 'Repository not found.' or
198
 
        (message.startswith('Repository ') and message.endswith(' not found.'))):
 
197
    if (message.startswith("Could not find Repository ")
 
198
        or message == 'Repository not found.'
 
199
            or (message.startswith('Repository ') and
 
200
                message.endswith(' not found.'))):
199
201
        return NotBranchError(url, message)
200
202
    if message == "HEAD failed to update":
201
203
        base_url, _ = urlutils.split_segment_parameters(url)
266
268
        if self._host == '':
267
269
            # return dulwich.client.LocalGitClient()
268
270
            return dulwich.client.SubprocessGitClient()
269
 
        return dulwich.client.TCPGitClient(self._host, self._port,
270
 
            report_activity=self._report_activity)
 
271
        return dulwich.client.TCPGitClient(
 
272
            self._host, self._port, report_activity=self._report_activity)
271
273
 
272
274
 
273
275
class SSHSocketWrapper(object):
292
294
        self.bzr_ssh_vendor = ssh._get_ssh_vendor()
293
295
 
294
296
    def run_command(self, host, command, username=None, port=None):
295
 
        connection = self.bzr_ssh_vendor.connect_ssh(username=username,
296
 
            password=None, port=port, host=host, command=command)
 
297
        connection = self.bzr_ssh_vendor.connect_ssh(
 
298
            username=username, password=None, port=port, host=host,
 
299
            command=command)
297
300
        (kind, io_object) = connection.get_sock_or_pipes()
298
301
        if kind == 'socket':
299
302
            return SSHSocketWrapper(io_object)
301
304
            raise AssertionError("Unknown io object kind %r'" % kind)
302
305
 
303
306
 
304
 
#dulwich.client.get_ssh_vendor = DulwichSSHVendor
 
307
# dulwich.client.get_ssh_vendor = DulwichSSHVendor
305
308
 
306
309
 
307
310
class SSHGitSmartTransport(GitSmartTransport):
320
323
            self._client = None
321
324
            return ret
322
325
        location_config = config.LocationConfig(self.base)
323
 
        client = dulwich.client.SSHGitClient(self._host, self._port, self._username,
 
326
        client = dulwich.client.SSHGitClient(
 
327
            self._host, self._port, self._username,
324
328
            report_activity=self._report_activity)
325
329
        # Set up alternate pack program paths
326
330
        upload_pack = location_config.get_user_option('git_upload_pack')
390
394
    def _gitrepository_class(self):
391
395
        return RemoteGitRepository
392
396
 
393
 
    def archive(self, format, committish, write_data, progress=None, write_error=None,
394
 
                subdirs=None, prefix=None):
395
 
        if format not in ('tar', 'zip'):
396
 
            raise errors.NoSuchExportFormat(format)
 
397
    def archive(self, format, committish, write_data, progress=None,
 
398
                write_error=None, subdirs=None, prefix=None):
397
399
        if progress is None:
398
400
            pb = ui.ui_factory.nested_progress_bar()
399
401
            progress = DefaultProgressReporter(pb).progress
400
402
        else:
401
403
            pb = None
 
404
        def progress_wrapper(message):
 
405
            if message.startswith(b"fatal: Unknown archive format \'"):
 
406
                format = message.strip()[len(b"fatal: Unknown archive format '"):-1]
 
407
                raise errors.NoSuchExportFormat(format.decode('ascii'))
 
408
            return progress(message)
402
409
        try:
403
 
            self._client.archive(self._client_path, committish,
404
 
                write_data, progress, write_error, format=format,
405
 
                subdirs=subdirs, prefix=prefix)
 
410
            self._client.archive(
 
411
                self._client_path, committish, write_data, progress_wrapper,
 
412
                write_error,
 
413
                format=(format.encode('ascii') if format else None),
 
414
                subdirs=subdirs,
 
415
                prefix=(prefix.encode('utf-8') if prefix else None))
406
416
        except GitProtocolError as e:
407
417
            raise parse_git_error(self.transport.external_url(), e)
408
418
        finally:
409
419
            if pb is not None:
410
420
                pb.finished()
411
421
 
412
 
    def fetch_pack(self, determine_wants, graph_walker, pack_data, progress=None):
 
422
    def fetch_pack(self, determine_wants, graph_walker, pack_data,
 
423
                   progress=None):
413
424
        if progress is None:
414
425
            pb = ui.ui_factory.nested_progress_bar()
415
426
            progress = DefaultProgressReporter(pb).progress
416
427
        else:
417
428
            pb = None
418
429
        try:
419
 
            result = self._client.fetch_pack(self._client_path, determine_wants,
420
 
                graph_walker, pack_data, progress)
 
430
            result = self._client.fetch_pack(
 
431
                self._client_path, determine_wants, graph_walker, pack_data,
 
432
                progress)
421
433
            if result.refs is None:
422
434
                result.refs = {}
423
 
            self._refs = remote_refs_dict_to_container(result.refs, result.symrefs)
 
435
            self._refs = remote_refs_dict_to_container(
 
436
                result.refs, result.symrefs)
424
437
            return result
425
438
        except GitProtocolError as e:
426
439
            raise parse_git_error(self.transport.external_url(), e)
434
447
            progress = DefaultProgressReporter(pb).progress
435
448
        else:
436
449
            pb = None
 
450
 
437
451
        def get_changed_refs_wrapper(refs):
438
452
            # TODO(jelmer): This drops symref information
439
453
            self._refs = remote_refs_dict_to_container(refs)
440
454
            return get_changed_refs(refs)
441
455
        try:
442
 
            return self._client.send_pack(self._client_path,
443
 
                    get_changed_refs_wrapper, generate_pack_data, progress)
 
456
            return self._client.send_pack(
 
457
                self._client_path, get_changed_refs_wrapper,
 
458
                generate_pack_data, progress)
444
459
        except GitProtocolError as e:
445
460
            raise parse_git_error(self.transport.external_url(), e)
446
461
        finally:
453
468
        if refname != b'HEAD' and refname in self.get_refs_container():
454
469
            raise AlreadyBranchError(self.user_url)
455
470
        if refname in self.get_refs_container():
456
 
            ref_chain, unused_sha = self.get_refs_container().follow(self._get_selected_ref(None))
 
471
            ref_chain, unused_sha = self.get_refs_container().follow(
 
472
                self._get_selected_ref(None))
457
473
            if ref_chain[0] == b'HEAD':
458
474
                refname = ref_chain[1]
459
475
        repo = self.open_repository()
461
477
 
462
478
    def destroy_branch(self, name=None):
463
479
        refname = self._get_selected_ref(name)
 
480
 
464
481
        def get_changed_refs(old_refs):
465
 
            ret = dict(old_refs)
466
 
            if not refname in ret:
 
482
            ret = {}
 
483
            if refname not in old_refs:
467
484
                raise NotBranchError(self.user_url)
468
485
            ret[refname] = dulwich.client.ZERO_SHA
469
486
            return ret
 
487
 
470
488
        def generate_pack_data(have, want, ofs_delta=False):
471
489
            return pack_objects_to_data([])
472
490
        self.send_pack(get_changed_refs, generate_pack_data)
490
508
    def open_repository(self):
491
509
        return RemoteGitRepository(self)
492
510
 
 
511
    def get_branch_reference(self, name=None):
 
512
        ref = branch_name_to_ref(name)
 
513
        val = self.get_refs_container().read_ref(ref)
 
514
        if val.startswith(SYMREF):
 
515
            return val[len(SYMREF):]
 
516
        return None
 
517
 
493
518
    def open_branch(self, name=None, unsupported=False,
494
 
            ignore_fallbacks=False, ref=None, possible_transports=None,
495
 
            nascent_ok=False):
 
519
                    ignore_fallbacks=False, ref=None, possible_transports=None,
 
520
                    nascent_ok=False):
496
521
        repo = self.open_repository()
497
522
        ref = self._get_selected_ref(name, ref)
498
 
        if not nascent_ok and ref not in self.get_refs_container():
 
523
        try:
 
524
            if not nascent_ok and ref not in self.get_refs_container():
 
525
                raise NotBranchError(
 
526
                    self.root_transport.base, controldir=self)
 
527
        except NotGitRepository:
499
528
            raise NotBranchError(self.root_transport.base,
500
 
                    controldir=self)
 
529
                                 controldir=self)
501
530
        ref_chain, unused_sha = self.get_refs_container().follow(ref)
502
531
        return RemoteGitBranch(self, repo, ref_chain[-1])
503
532
 
514
543
        if self._refs is not None:
515
544
            return self._refs
516
545
        result = self.fetch_pack(lambda x: None, None,
517
 
            lambda x: None, lambda x: trace.mutter("git: %s" % x))
 
546
                                 lambda x: None,
 
547
                                 lambda x: trace.mutter("git: %s" % x))
518
548
        self._refs = remote_refs_dict_to_container(
519
 
                result.refs, result.symrefs)
 
549
            result.refs, result.symrefs)
520
550
        return self._refs
521
551
 
522
552
    def push_branch(self, source, revision_id=None, overwrite=False,
539
569
        if isinstance(source, GitBranch) and lossy:
540
570
            raise errors.LossyPushToSameVCS(source.controldir, self)
541
571
        source_store = get_object_store(source.repository)
 
572
        fetch_tags = source.get_config_stack().get('branch.fetch_tags')
 
573
        def get_changed_refs(refs):
 
574
            self._refs = remote_refs_dict_to_container(refs)
 
575
            ret = {}
 
576
            # TODO(jelmer): Unpeel if necessary
 
577
            push_result.new_original_revid = revision_id
 
578
            if lossy:
 
579
                new_sha = source_store._lookup_revision_sha1(revision_id)
 
580
            else:
 
581
                try:
 
582
                    new_sha = repo.lookup_bzr_revision_id(revision_id)[0]
 
583
                except errors.NoSuchRevision:
 
584
                    raise errors.NoRoundtrippingSupport(
 
585
                        source, self.open_branch(name=name, nascent_ok=True))
 
586
            if not overwrite:
 
587
                if remote_divergence(ret.get(refname), new_sha,
 
588
                                     source_store):
 
589
                    raise DivergedBranches(
 
590
                        source, self.open_branch(name, nascent_ok=True))
 
591
            ret[refname] = new_sha
 
592
            if fetch_tags:
 
593
                for tagname, revid in viewitems(source.tags.get_tag_dict()):
 
594
                    if lossy:
 
595
                        new_sha = source_store._lookup_revision_sha1(revid)
 
596
                    else:
 
597
                        try:
 
598
                            new_sha = repo.lookup_bzr_revision_id(revid)[0]
 
599
                        except errors.NoSuchRevision:
 
600
                            continue
 
601
                    ret[tag_name_to_ref(tagname)] = new_sha
 
602
            return ret
542
603
        with source_store.lock_read():
543
 
            def get_changed_refs(refs):
544
 
                self._refs = remote_refs_dict_to_container(refs)
545
 
                ret = dict(refs)
546
 
                # TODO(jelmer): Unpeel if necessary
547
 
                push_result.new_original_revid = revision_id
548
 
                if lossy:
549
 
                    new_sha = source_store._lookup_revision_sha1(revision_id)
550
 
                else:
551
 
                    new_sha = repo.lookup_bzr_revision_id(revision_id)[0]
552
 
                if not overwrite:
553
 
                    if remote_divergence(ret.get(refname), new_sha, source_store):
554
 
                        raise DivergedBranches(
555
 
                                source, self.open_branch(name, nascent_ok=True))
556
 
                ret[refname] = new_sha
557
 
                return ret
558
604
            if lossy:
559
605
                generate_pack_data = source_store.generate_lossy_pack_data
560
606
            else:
561
607
                generate_pack_data = source_store.generate_pack_data
562
608
            new_refs = self.send_pack(get_changed_refs, generate_pack_data)
563
609
        push_result.new_revid = repo.lookup_foreign_revision_id(
564
 
                new_refs[refname])
 
610
            new_refs[refname])
565
611
        try:
566
612
            old_remote = self._refs[refname]
567
613
        except KeyError:
572
618
        if old_remote != ZERO_SHA:
573
619
            push_result.branch_push_result = GitBranchPushResult()
574
620
            push_result.branch_push_result.source_branch = source
575
 
            push_result.branch_push_result.target_branch = push_result.target_branch
 
621
            push_result.branch_push_result.target_branch = (
 
622
                push_result.target_branch)
576
623
            push_result.branch_push_result.local_branch = None
577
 
            push_result.branch_push_result.master_branch = push_result.target_branch
 
624
            push_result.branch_push_result.master_branch = (
 
625
                push_result.target_branch)
578
626
            push_result.branch_push_result.old_revid = push_result.old_revid
579
627
            push_result.branch_push_result.new_revid = push_result.new_revid
580
 
            push_result.branch_push_result.new_original_revid = push_result.new_original_revid
 
628
            push_result.branch_push_result.new_original_revid = (
 
629
                push_result.new_original_revid)
581
630
        if source.get_push_location() is None or remember:
582
631
            source.set_push_location(push_result.target_branch.base)
583
632
        return push_result
607
656
                def report_progress(cur, total):
608
657
                    pb.update("generating index", cur, total)
609
658
                self.data.create_index(path,
610
 
                    progress=report_progress)
 
659
                                       progress=report_progress)
611
660
            finally:
612
661
                pb.finished()
613
662
        return load_pack_index(path)
625
674
 
626
675
    def __init__(self, transport, *args, **kwargs):
627
676
        self.transport = transport
628
 
        super(BzrGitHttpClient, self).__init__(transport.external_url(), *args, **kwargs)
 
677
        url = urlutils.URL.from_string(transport.external_url())
 
678
        url.user = url.quoted_user = None
 
679
        url.password = url.quoted_password = None
 
680
        super(BzrGitHttpClient, self).__init__(str(url), *args, **kwargs)
629
681
 
630
682
    def _http_request(self, url, headers=None, data=None,
631
683
                      allow_compression=False):
640
692
            `redirect_location` properties, and `read` is a consumable read
641
693
            method for the response data.
642
694
        """
643
 
        from breezy.transport.http._urllib2_wrappers import Request
644
695
        headers['User-agent'] = user_agent_for_github()
645
696
        headers["Pragma"] = "no-cache"
646
697
        if allow_compression:
648
699
        else:
649
700
            headers["Accept-Encoding"] = "identity"
650
701
 
651
 
        request = Request(
 
702
        response = self.transport.request(
652
703
            ('GET' if data is None else 'POST'),
653
 
            url, data, headers,
654
 
            accepted_errors=[200, 404])
655
 
        request.follow_redirections = True
656
 
 
657
 
        response = self.transport._perform(request)
658
 
 
659
 
        if response.code == 404:
 
704
            body=data,
 
705
            headers=headers, retries=8)
 
706
 
 
707
        if response.status == 404:
660
708
            raise NotGitRepository()
661
 
        elif response.code != 200:
 
709
        elif response.status != 200:
662
710
            raise GitProtocolError("unexpected http resp %d for %s" %
663
711
                                   (response.code, url))
664
712
 
680
728
                self.content_type = response.getheader("Content-Type")
681
729
                self.redirect_location = response.geturl()
682
730
 
 
731
            def readlines(self):
 
732
                return self._response.readlines()
 
733
 
683
734
            def close(self):
684
735
                self._response.close()
685
736
 
725
776
        else:
726
777
            raise NotBranchError(transport.base)
727
778
        if not _found:
728
 
            pass # TODO(jelmer): Actually probe for something
 
779
            pass  # TODO(jelmer): Actually probe for something
729
780
        return RemoteGitDir(transport, self, client, client_path)
730
781
 
731
782
    def get_format_description(self):
739
790
            external_url = transport.external_url()
740
791
        except InProcessTransport:
741
792
            raise NotBranchError(path=transport.base)
742
 
        return (external_url.startswith("http:") or
743
 
                external_url.startswith("https:") or
744
 
                external_url.startswith("git+") or
745
 
                external_url.startswith("git:"))
 
793
        return (external_url.startswith("http:")
 
794
                or external_url.startswith("https:")
 
795
                or external_url.startswith("git+")
 
796
                or external_url.startswith("git:"))
746
797
 
747
798
 
748
799
class GitRemoteRevisionTree(RevisionTree):
762
813
        # git-upload-archive(1) generaly only supports refs. So let's see if we
763
814
        # can find one.
764
815
        reverse_refs = {
765
 
                v: k for (k, v) in
766
 
                self._repository.controldir.get_refs_container().as_dict().items()}
 
816
            v: k for (k, v) in
 
817
            self._repository.controldir.get_refs_container().as_dict().items()}
767
818
        try:
768
819
            committish = reverse_refs[commit]
769
820
        except KeyError:
771
822
            # Let's hope for the best.
772
823
            committish = commit
773
824
        self._repository.archive(
774
 
                format, committish, f.write,
775
 
                subdirs=([subdir] if subdir else None),
776
 
                prefix=(root+'/') if root else '')
 
825
            format, committish, f.write,
 
826
            subdirs=([subdir] if subdir else None),
 
827
            prefix=(root + '/') if root else '')
777
828
        f.seek(0)
778
829
        return osutils.file_iterator(f)
779
830
 
 
831
    def is_versioned(self, path):
 
832
        raise GitSmartRemoteNotSupported(self.is_versioned, self)
 
833
 
 
834
    def has_filename(self, path):
 
835
        raise GitSmartRemoteNotSupported(self.has_filename, self)
 
836
 
 
837
    def get_file_text(self, path):
 
838
        raise GitSmartRemoteNotSupported(self.get_file_text, self)
 
839
 
780
840
 
781
841
class RemoteGitRepository(GitRepository):
782
842
 
 
843
    supports_random_access = False
 
844
 
783
845
    @property
784
846
    def user_url(self):
785
847
        return self.control_url
792
854
 
793
855
    def fetch_pack(self, determine_wants, graph_walker, pack_data,
794
856
                   progress=None):
795
 
        return self.controldir.fetch_pack(determine_wants, graph_walker,
796
 
                                          pack_data, progress)
 
857
        return self.controldir.fetch_pack(
 
858
            determine_wants, graph_walker, pack_data, progress)
797
859
 
798
860
    def send_pack(self, get_changed_refs, generate_pack_data):
799
861
        return self.controldir.send_pack(get_changed_refs, generate_pack_data)
803
865
        fd, path = tempfile.mkstemp(suffix=".pack")
804
866
        try:
805
867
            self.fetch_pack(determine_wants, graph_walker,
806
 
                lambda x: os.write(fd, x), progress)
 
868
                            lambda x: os.write(fd, x), progress)
807
869
        finally:
808
870
            os.close(fd)
809
871
        if os.path.getsize(path) == 0:
811
873
        return TemporaryPackIterator(path[:-len(".pack")], resolve_ext_ref)
812
874
 
813
875
    def lookup_bzr_revision_id(self, bzr_revid, mapping=None):
814
 
        # This won't work for any round-tripped bzr revisions, but it's a start..
 
876
        # This won't work for any round-tripped bzr revisions, but it's a
 
877
        # start..
815
878
        try:
816
879
            return mapping_registry.revision_id_bzr_to_foreign(bzr_revid)
817
880
        except InvalidRevisionId:
847
910
 
848
911
    def _set_ref(self, name, sha):
849
912
        ref = tag_name_to_ref(name)
 
913
 
850
914
        def get_changed_refs(old_refs):
851
 
            ret = dict(old_refs)
852
 
            if sha == dulwich.client.ZERO_SHA and ref not in ret:
 
915
            ret = {}
 
916
            if sha == dulwich.client.ZERO_SHA and ref not in old_refs:
853
917
                raise NoSuchTag(name)
854
918
            ret[ref] = sha
855
919
            return ret
 
920
 
856
921
        def generate_pack_data(have, want, ofs_delta=False):
857
922
            return pack_objects_to_data([])
858
923
        self.repository.send_pack(get_changed_refs, generate_pack_data)
863
928
    def __init__(self, controldir, repository, name):
864
929
        self._sha = None
865
930
        super(RemoteGitBranch, self).__init__(controldir, repository, name,
866
 
                RemoteGitBranchFormat())
 
931
                                              RemoteGitBranchFormat())
867
932
 
868
933
    def last_revision_info(self):
869
934
        raise GitSmartRemoteNotSupported(self.last_revision_info, self)
896
961
 
897
962
    def _synchronize_history(self, destination, revision_id):
898
963
        """See Branch._synchronize_history()."""
899
 
        destination.generate_revision_history(self.last_revision())
 
964
        if revision_id is None:
 
965
            revision_id = self.last_revision()
 
966
        destination.generate_revision_history(revision_id)
900
967
 
901
968
    def _get_parent_location(self):
902
969
        return None
927
994
                raise TypeError(tag_name)
928
995
            yield (ref_name, tag_name, peeled, unpeeled)
929
996
 
 
997
    def set_last_revision_info(self, revno, revid):
 
998
        self.generate_revision_history(revid)
 
999
 
 
1000
    def generate_revision_history(self, revision_id, last_rev=None,
 
1001
                                  other_branch=None):
 
1002
        sha = self.lookup_bzr_revision_id(revision_id)[0]
 
1003
        def get_changed_refs(old_refs):
 
1004
            return {self.ref: sha}
 
1005
        def generate_pack_data(have, want, ofs_delta=False):
 
1006
            return pack_objects_to_data([])
 
1007
        self.repository.send_pack(get_changed_refs, generate_pack_data)
 
1008
        self._sha = sha
 
1009
 
930
1010
 
931
1011
def remote_refs_dict_to_container(refs_dict, symrefs_dict={}):
932
1012
    base = {}