/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/transport/http/_urllib2_wrappers.py

  • Committer: Jelmer Vernooij
  • Date: 2018-11-16 19:47:19 UTC
  • mfrom: (7178 work)
  • mto: This revision was merged to the branch mainline in revision 7179.
  • Revision ID: jelmer@jelmer.uk-20181116194719-m5ut2wfuze5x9s1p
Merge trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/transport/http/_urllib2_wrappers.py
156
156
    else:
157
157
        return u'required'
158
158
 
 
159
 
159
160
opt_ssl_ca_certs = config.Option('ssl.ca_certs',
160
161
                                 from_unicode=ca_certs_from_store,
161
162
                                 default=default_ca_certs,
283
284
    # 8k chunks should be fine.
284
285
    _discarded_buf_size = 8192
285
286
 
 
287
    if PY3:
 
288
        def __init__(self, sock, debuglevel=0, method=None, url=None):
 
289
            self.url = url
 
290
            super(Response, self).__init__(
 
291
                sock, debuglevel=debuglevel, method=method, url=url)
 
292
 
286
293
    def begin(self):
287
294
        """Begin to read the response from the server.
288
295
 
380
387
                pending = self._response.finish()
381
388
                # Warn the user (once)
382
389
                if (self._ranges_received_whole_file is None
383
 
                    and self._response.status == 200
384
 
                    and pending and pending > self._range_warning_thresold
385
 
                    ):
 
390
                        and self._response.status == 200
 
391
                        and pending
 
392
                        and pending > self._range_warning_thresold):
386
393
                    self._ranges_received_whole_file = True
387
394
                    trace.warning(
388
395
                        'Got a 200 response when asking for multiple ranges,'
394
401
                # cleaning anymore, so no need to fail, we just get rid of the
395
402
                # socket and let callers reconnect
396
403
                if (len(e.args) == 0
397
 
                    or e.args[0] not in (errno.ECONNRESET, errno.ECONNABORTED)):
 
404
                        or e.args[0] not in (errno.ECONNRESET, errno.ECONNABORTED)):
398
405
                    raise
399
406
                self.close()
400
407
            self._response = None
440
447
        AbstractHTTPConnection.__init__(self, report_activity=report_activity)
441
448
        if PY3:
442
449
            http_client.HTTPSConnection.__init__(
443
 
                    self, host, port, key_file, cert_file)
 
450
                self, host, port, key_file, cert_file)
444
451
        else:
445
452
            # Use strict=True since we don't support HTTP/0.9
446
453
            http_client.HTTPSConnection.__init__(self, host, port,
447
 
                                             key_file, cert_file, strict=True)
 
454
                                                 key_file, cert_file, strict=True)
448
455
        self.proxied_host = proxied_host
449
456
        self.ca_certs = ca_certs
450
457
 
517
524
                 connection=None, parent=None,
518
525
                 accepted_errors=None):
519
526
        urllib_request.Request.__init__(
520
 
                self, url, data, headers,
521
 
                origin_req_host, unverifiable)
 
527
            self, url, data, headers,
 
528
            origin_req_host, unverifiable)
522
529
        self.method = method
523
530
        self.connection = connection
524
531
        self.accepted_errors = accepted_errors
607
614
    http[s] requests in AbstractHTTPHandler.
608
615
    """
609
616
 
610
 
    handler_order = 1000 # after all pre-processings
 
617
    handler_order = 1000  # after all pre-processings
611
618
 
612
619
    def __init__(self, report_activity=None, ca_certs=None):
613
620
        self._report_activity = report_activity
630
637
        except http_client.InvalidURL as exception:
631
638
            # There is only one occurrence of InvalidURL in http_client
632
639
            raise urlutils.InvalidURL(request.get_full_url(),
633
 
                                    extra='nonnumeric port')
 
640
                                      extra='nonnumeric port')
634
641
 
635
642
        return connection
636
643
 
715
722
        exc_type, exc_val, exc_tb = sys.exc_info()
716
723
        if exc_type == socket.gaierror:
717
724
            # No need to retry, that will not help
 
725
            if PY3:
 
726
                origin_req_host = request.origin_req_host
 
727
            else:
 
728
                origin_req_host = request.get_origin_req_host()
718
729
            raise errors.ConnectionError("Couldn't resolve host '%s'"
719
 
                                         % request.get_origin_req_host(),
 
730
                                         % origin_req_host,
720
731
                                         orig_error=exc_val)
721
732
        elif isinstance(exc_val, http_client.ImproperConnectionState):
722
733
            # The http_client pipeline is in incorrect state, it's a bug in our
749
760
                        orig_error=exc_val)
750
761
                elif (isinstance(exc_val, socket.error) and len(exc_val.args)
751
762
                      and exc_val.args[0] in (errno.ECONNRESET, 10053, 10054)):
752
 
                      # 10053 == WSAECONNABORTED
753
 
                      # 10054 == WSAECONNRESET
 
763
                    # 10053 == WSAECONNABORTED
 
764
                    # 10054 == WSAECONNRESET
754
765
                    raise errors.ConnectionReset(
755
766
                        "Connection lost while sending request.")
756
767
                else:
765
776
                    else:
766
777
                        selector = request.get_selector()
767
778
                    my_exception = errors.ConnectionError(
768
 
                        msg= 'while sending %s %s:' % (request.get_method(),
769
 
                                                       selector),
 
779
                        msg='while sending %s %s:' % (request.get_method(),
 
780
                                                      selector),
770
781
                        orig_error=exc_val)
771
782
 
772
783
                if self._debuglevel >= 2:
809
820
            if sys.version_info[:2] >= (3, 6):
810
821
                connection._send_request(method, url,
811
822
                                         # FIXME: implements 100-continue
812
 
                                         #None, # We don't send the body yet
 
823
                                         # None, # We don't send the body yet
813
824
                                         request.data,
814
825
                                         headers, encode_chunked=False)
815
826
            else:
816
827
                connection._send_request(method, url,
817
828
                                         # FIXME: implements 100-continue
818
 
                                         #None, # We don't send the body yet
 
829
                                         # None, # We don't send the body yet
819
830
                                         request.data,
820
831
                                         headers)
821
832
            if 'http' in debug.debug_flags:
829
840
                    hdrs.append('%s: %s' % (k, v))
830
841
                trace.mutter('> ' + '\n> '.join(hdrs) + '\n')
831
842
            if self._debuglevel >= 1:
832
 
                print('Request sent: [%r] from (%s)' \
833
 
                    % (request, request.connection.sock.getsockname()))
 
843
                print('Request sent: [%r] from (%s)'
 
844
                      % (request, request.connection.sock.getsockname()))
834
845
            response = connection.getresponse()
835
846
            convert_to_addinfourl = True
836
847
        except (ssl.SSLError, ssl.CertificateError):
911
922
        connection = request.connection
912
923
        if connection.sock is None and \
913
924
                connection.proxied_host is not None and \
914
 
                request.get_method() != 'CONNECT' : # Don't loop
 
925
                request.get_method() != 'CONNECT':  # Don't loop
915
926
            # FIXME: We need a gazillion connection tests here, but we still
916
927
            # miss a https server :-( :
917
928
            # - with and without proxy
930
941
            response = self.parent.open(connect)
931
942
            if response.code != 200:
932
943
                raise errors.ConnectionError("Can't connect to %s via proxy %s" % (
933
 
                        connect.proxied_host, self.host))
 
944
                    connect.proxied_host, self.host))
934
945
            # Housekeeping
935
946
            connection.cleanup_pipe()
936
947
            # Establish the connection encryption
939
950
            request.connection = connection
940
951
        return self.do_open(HTTPSConnection, request)
941
952
 
 
953
 
942
954
class HTTPRedirectHandler(urllib_request.HTTPRedirectHandler):
943
955
    """Handles redirect requests.
944
956
 
992
1004
 
993
1005
        if code in (301, 302, 303, 307):
994
1006
            return Request(req.get_method(), newurl,
995
 
                           headers = req.headers,
996
 
                           origin_req_host = origin_req_host,
997
 
                           unverifiable = True,
 
1007
                           headers=req.headers,
 
1008
                           origin_req_host=origin_req_host,
 
1009
                           unverifiable=True,
998
1010
                           # TODO: It will be nice to be able to
999
1011
                           # detect virtual hosts sharing the same
1000
1012
                           # IP address, that will allow us to
1001
1013
                           # share the same connection...
1002
 
                           connection = None,
1003
 
                           parent = req,
 
1014
                           connection=None,
 
1015
                           parent=req,
1004
1016
                           )
1005
1017
        else:
1006
 
            raise urllib_request.HTTPError(req.get_full_url(), code, msg, headers, fp)
 
1018
            raise urllib_request.HTTPError(
 
1019
                req.get_full_url(), code, msg, headers, fp)
1007
1020
 
1008
1021
    def http_error_302(self, req, fp, code, msg, headers):
1009
1022
        """Requests the redirected to URI.
1044
1057
        if hasattr(req, 'redirect_dict'):
1045
1058
            visited = redirected_req.redirect_dict = req.redirect_dict
1046
1059
            if (visited.get(newurl, 0) >= self.max_repeats or
1047
 
                len(visited) >= self.max_redirections):
 
1060
                    len(visited) >= self.max_redirections):
1048
1061
                raise urllib_request.HTTPError(req.get_full_url(), code,
1049
 
                                        self.inf_msg + msg, headers, fp)
 
1062
                                               self.inf_msg + msg, headers, fp)
1050
1063
        else:
1051
1064
            visited = redirected_req.redirect_dict = req.redirect_dict = {}
1052
1065
        visited[newurl] = visited.get(newurl, 0) + 1
1098
1111
            if self._debuglevel >= 3:
1099
1112
                print('Will bind %s for %r' % (scheme_request, proxy))
1100
1113
            setattr(self, scheme_request,
1101
 
                lambda request: self.set_proxy(request, scheme))
 
1114
                    lambda request: self.set_proxy(request, scheme))
1102
1115
        # We are interested only by the http[s] proxies
1103
1116
        http_proxy = self.get_proxy_env_var('http')
1104
1117
        bind_scheme_request(http_proxy, 'http')
1194
1207
            # proxied request, intialize.  scheme (the authentication scheme)
1195
1208
            # and realm will be set by the AuthHandler
1196
1209
            request.proxy_auth = {
1197
 
                                  'host': parsed_url.host,
1198
 
                                  'port': parsed_url.port,
1199
 
                                  'user': parsed_url.user,
1200
 
                                  'password': parsed_url.password,
1201
 
                                  'protocol': parsed_url.scheme,
1202
 
                                   # We ignore path since we connect to a proxy
1203
 
                                  'path': None}
 
1210
                'host': parsed_url.host,
 
1211
                'port': parsed_url.port,
 
1212
                'user': parsed_url.user,
 
1213
                'password': parsed_url.password,
 
1214
                'protocol': parsed_url.scheme,
 
1215
                # We ignore path since we connect to a proxy
 
1216
                'path': None}
1204
1217
        if parsed_url.port is None:
1205
1218
            phost = parsed_url.host
1206
1219
        else:
1338
1351
                # auth_match may have modified auth (by adding the
1339
1352
                # password or changing the realm, for example)
1340
1353
                if (request.get_header(self.auth_header, None) is not None
1341
 
                    and not auth['modified']):
 
1354
                        and not auth['modified']):
1342
1355
                    # We already tried that, give up
1343
1356
                    return None
1344
1357
 
1353
1366
                    # the credentials are wrong (or incomplete), but we know
1354
1367
                    # that the associated scheme should be used.
1355
1368
                    best_scheme = auth['best_scheme'] = self.scheme
1356
 
                if  best_scheme != self.scheme:
 
1369
                if best_scheme != self.scheme:
1357
1370
                    continue
1358
1371
 
1359
1372
                if self.requires_username and auth.get('user', None) is None:
1487
1500
        """Insert an authentication header if information is available"""
1488
1501
        auth = self.get_auth(request)
1489
1502
        if self.auth_params_reusable(auth):
1490
 
            self.add_auth_header(request, self.build_auth_header(auth, request))
 
1503
            self.add_auth_header(
 
1504
                request, self.build_auth_header(auth, request))
1491
1505
        return request
1492
1506
 
1493
 
    https_request = http_request # FIXME: Need test
 
1507
    https_request = http_request  # FIXME: Need test
1494
1508
 
1495
1509
 
1496
1510
class NegotiateAuthHandler(AbstractAuthHandler):
1530
1544
        ret, vc = kerberos.authGSSClientInit("HTTP@%(host)s" % auth)
1531
1545
        if ret < 1:
1532
1546
            trace.warning('Unable to create GSSAPI context for %s: %d',
1533
 
                auth['host'], ret)
 
1547
                          auth['host'], ret)
1534
1548
            return None
1535
1549
        ret = kerberos.authGSSClientStep(vc, "")
1536
1550
        if ret < 0:
1558
1572
 
1559
1573
    def build_auth_header(self, auth, request):
1560
1574
        raw = '%s:%s' % (auth['user'], auth['password'])
1561
 
        auth_header = 'Basic ' + base64.b64encode(raw.encode('utf-8')).decode('ascii')
 
1575
        auth_header = 'Basic ' + \
 
1576
            base64.b64encode(raw.encode('utf-8')).decode('ascii')
1562
1577
        return auth_header
1563
1578
 
1564
1579
    def extract_realm(self, header_value):
1579
1594
            self.update_auth(auth, 'scheme', scheme)
1580
1595
            self.update_auth(auth, 'realm', realm)
1581
1596
            if (auth.get('user', None) is None
1582
 
                or auth.get('password', None) is None):
 
1597
                    or auth.get('password', None) is None):
1583
1598
                user, password = self.get_user_password(auth)
1584
1599
                self.update_auth(auth, 'user', user)
1585
1600
                self.update_auth(auth, 'password', password)
1596
1611
    H = None
1597
1612
    KD = None
1598
1613
    if algorithm == 'MD5':
1599
 
        H = lambda x: osutils.md5(x).hexdigest()
 
1614
        def H(x): return osutils.md5(x).hexdigest()
1600
1615
    elif algorithm == 'SHA':
1601
1616
        H = osutils.sha_string
1602
1617
    if H is not None:
1603
 
        KD = lambda secret, data: H(("%s:%s" % (secret, data)).encode('utf-8'))
 
1618
        def KD(secret, data): return H(
 
1619
            ("%s:%s" % (secret, data)).encode('utf-8'))
1604
1620
    return H, KD
1605
1621
 
1606
1622
 
1629
1645
            return False
1630
1646
 
1631
1647
        # Put the requested authentication info into a dict
1632
 
        req_auth = urllib_request.parse_keqv_list(urllib_request.parse_http_list(raw_auth))
 
1648
        req_auth = urllib_request.parse_keqv_list(
 
1649
            urllib_request.parse_http_list(raw_auth))
1633
1650
 
1634
1651
        # Check that we can handle that authentication
1635
1652
        qop = req_auth.get('qop', None)
1636
 
        if qop != 'auth': # No auth-int so far
 
1653
        if qop != 'auth':  # No auth-int so far
1637
1654
            return False
1638
1655
 
1639
1656
        H, KD = get_digest_algorithm_impls(req_auth.get('algorithm', 'MD5'))
1673
1690
        url_scheme, url_selector = splittype(selector)
1674
1691
        sel_host, uri = splithost(url_selector)
1675
1692
 
1676
 
        A1 = ('%s:%s:%s' % (auth['user'], auth['realm'], auth['password'])).encode('utf-8')
 
1693
        A1 = ('%s:%s:%s' %
 
1694
              (auth['user'], auth['realm'], auth['password'])).encode('utf-8')
1677
1695
        A2 = ('%s:%s' % (request.get_method(), uri)).encode('utf-8')
1678
1696
 
1679
1697
        nonce = auth['nonce']
1804
1822
    instead, we leave our Transport handle them.
1805
1823
    """
1806
1824
 
1807
 
    accepted_errors = [200, # Ok
1808
 
                       206, # Partial content
1809
 
                       404, # Not found
 
1825
    accepted_errors = [200,  # Ok
 
1826
                       206,  # Partial content
 
1827
                       404,  # Not found
1810
1828
                       ]
1811
1829
    """The error codes the caller will handle.
1812
1830