/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/tests/test_http.py

Merge test-run support.

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/tests/test_http.py
24
24
# TODO: What about renaming to breezy.tests.transport.http ?
25
25
 
26
26
try:
27
 
    from http.client import UnknownProtocol, parse_headers
 
27
    from http.client import UnknownProtocol
28
28
    from http.server import SimpleHTTPRequestHandler
29
29
except ImportError:  # python < 3
30
30
    from httplib import UnknownProtocol
50
50
from ..bzr import (
51
51
    remote as _mod_remote,
52
52
    )
53
 
from ..sixish import PY3
54
53
from . import (
55
54
    features,
56
55
    http_server,
67
66
    )
68
67
from ..transport.http import (
69
68
    HttpTransport,
 
69
    _urllib2_wrappers,
70
70
    )
71
71
 
72
72
 
86
86
def vary_by_http_protocol_version():
87
87
    """Test on http/1.0 and 1.1"""
88
88
    return [
89
 
        ('HTTP/1.0', dict(_protocol_version='HTTP/1.0')),
90
 
        ('HTTP/1.1', dict(_protocol_version='HTTP/1.1')),
 
89
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
90
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
91
91
        ]
92
92
 
93
93
 
124
124
def vary_by_http_activity():
125
125
    activity_scenarios = [
126
126
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
127
 
                             _transport=HttpTransport,)),
 
127
                            _transport=HttpTransport,)),
128
128
        ]
129
129
    if features.HTTPSServerFeature.available():
130
130
        # FIXME: Until we have a better way to handle self-signed certificates
134
134
        from . import (
135
135
            ssl_certs,
136
136
            )
137
 
 
138
137
        class HTTPS_transport(HttpTransport):
139
138
 
140
139
            def __init__(self, base, _from_transport=None):
173
172
        self._expect_body_tail = expect_body_tail
174
173
        self.host = None
175
174
        self.port = None
176
 
        self.received_bytes = b''
 
175
        self.received_bytes = ''
177
176
        self.scheme = scheme
178
177
 
179
178
    def get_url(self):
198
197
        if self._expect_body_tail is not None:
199
198
            while not self.received_bytes.endswith(self._expect_body_tail):
200
199
                self.received_bytes += conn.recv(4096)
201
 
            conn.sendall(b'HTTP/1.1 200 OK\r\n')
 
200
            conn.sendall('HTTP/1.1 200 OK\r\n')
202
201
        try:
203
202
            self._sock.close()
204
203
        except socket.error:
225
224
 
226
225
    def parse_header(self, header, auth_handler_class=None):
227
226
        if auth_handler_class is None:
228
 
            auth_handler_class = http.AbstractAuthHandler
229
 
        self.auth_handler = auth_handler_class()
 
227
            auth_handler_class = _urllib2_wrappers.AbstractAuthHandler
 
228
        self.auth_handler =  auth_handler_class()
230
229
        return self.auth_handler._parse_auth_header(header)
231
230
 
232
231
    def test_empty_header(self):
246
245
        self.assertEqual('realm="Thou should not pass"', remainder)
247
246
 
248
247
    def test_build_basic_header_with_long_creds(self):
249
 
        handler = http.BasicAuthHandler()
 
248
        handler = _urllib2_wrappers.BasicAuthHandler()
250
249
        user = 'user' * 10  # length 40
251
250
        password = 'password' * 5  # length 40
252
251
        header = handler.build_auth_header(
258
257
    def test_basic_extract_realm(self):
259
258
        scheme, remainder = self.parse_header(
260
259
            'Basic realm="Thou should not pass"',
261
 
            http.BasicAuthHandler)
 
260
            _urllib2_wrappers.BasicAuthHandler)
262
261
        match, realm = self.auth_handler.extract_realm(remainder)
263
262
        self.assertTrue(match is not None)
264
 
        self.assertEqual(u'Thou should not pass', realm)
 
263
        self.assertEqual('Thou should not pass', realm)
265
264
 
266
265
    def test_digest_header(self):
267
266
        scheme, remainder = self.parse_header(
275
274
    def setUp(self):
276
275
        super(TestHTTPRangeParsing, self).setUp()
277
276
        # We focus on range  parsing here and ignore everything else
278
 
 
279
277
        class RequestHandler(http_server.TestingHTTPRequestHandler):
280
278
            def setup(self): pass
281
 
 
282
279
            def handle(self): pass
283
 
 
284
280
            def finish(self): pass
285
281
 
286
282
        self.req_handler = RequestHandler(None, None, None)
287
283
 
288
284
    def assertRanges(self, ranges, header, file_size):
289
285
        self.assertEqual(ranges,
290
 
                         self.req_handler._parse_ranges(header, file_size))
 
286
                          self.req_handler._parse_ranges(header, file_size))
291
287
 
292
288
    def test_simple_range(self):
293
289
        self.assertRanges([(0, 2)], 'bytes=0-2', 12)
392
388
        self._transport('http://example.com/bzr/bzr.dev/')
393
389
        self.assertRaises(urlutils.InvalidURL,
394
390
                          self._transport,
395
 
                          'http://example.com:port/bzr/bzr.dev/')
 
391
                          'http://http://example.com/bzr/bzr.dev/')
396
392
 
397
393
    def test_http_root_urls(self):
398
394
        """Construction of URLs from server root"""
431
427
        self.assertEqual(t.has('foo/bar'), True)
432
428
        self.assertEqual(len(server.logs), 1)
433
429
        self.assertContainsRe(server.logs[0],
434
 
                              r'"HEAD /foo/bar HTTP/1.." (200|302) - "-" "Breezy/')
 
430
            r'"HEAD /foo/bar HTTP/1.." (200|302) - "-" "Breezy/')
435
431
 
436
432
    def test_http_has_not_found(self):
437
433
        server = self.get_readonly_server()
438
434
        t = self.get_readonly_transport()
439
435
        self.assertEqual(t.has('not-found'), False)
440
436
        self.assertContainsRe(server.logs[1],
441
 
                              r'"HEAD /not-found HTTP/1.." 404 - "-" "Breezy/')
 
437
            r'"HEAD /not-found HTTP/1.." 404 - "-" "Breezy/')
442
438
 
443
439
    def test_http_get(self):
444
440
        server = self.get_readonly_server()
446
442
        fp = t.get('foo/bar')
447
443
        self.assertEqualDiff(
448
444
            fp.read(),
449
 
            b'contents of foo/bar\n')
 
445
            'contents of foo/bar\n')
450
446
        self.assertEqual(len(server.logs), 1)
451
447
        self.assertTrue(server.logs[0].find(
452
448
            '"GET /foo/bar HTTP/1.1" 200 - "-" "Breezy/%s'
487
483
        )
488
484
 
489
485
    def test_post_body_is_received(self):
490
 
        server = RecordingServer(expect_body_tail=b'end-of-body',
 
486
        server = RecordingServer(expect_body_tail='end-of-body',
491
487
                                 scheme=self._url_protocol)
492
488
        self.start_server(server)
493
489
        url = server.get_url()
494
490
        # FIXME: needs a cleanup -- vila 20100611
495
491
        http_transport = transport.get_transport_from_url(url)
496
 
        code, response = http_transport._post(b'abc def end-of-body')
497
 
        self.assertTrue(
498
 
            server.received_bytes.startswith(b'POST /.bzr/smart HTTP/1.'))
499
 
        self.assertTrue(
500
 
            b'content-length: 19\r' in server.received_bytes.lower())
501
 
        self.assertTrue(b'content-type: application/octet-stream\r'
 
492
        code, response = http_transport._post('abc def end-of-body')
 
493
        self.assertTrue(
 
494
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
 
495
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
 
496
        self.assertTrue('content-type: application/octet-stream\r'
502
497
                        in server.received_bytes.lower())
503
498
        # The transport should not be assuming that the server can accept
504
499
        # chunked encoding the first time it connects, because HTTP/1.1, so we
505
500
        # check for the literal string.
506
501
        self.assertTrue(
507
 
            server.received_bytes.endswith(b'\r\n\r\nabc def end-of-body'))
 
502
            server.received_bytes.endswith('\r\n\r\nabc def end-of-body'))
508
503
 
509
504
 
510
505
class TestRangeHeader(tests.TestCase):
511
506
    """Test range_header method"""
512
507
 
513
508
    def check_header(self, value, ranges=[], tail=0):
514
 
        offsets = [(start, end - start + 1) for start, end in ranges]
 
509
        offsets = [ (start, end - start + 1) for start, end in ranges]
515
510
        coalesce = transport.Transport._coalesce_offsets
516
511
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
517
512
        range_header = http.HttpTransport._range_header
633
628
    def parse_request(self):
634
629
        """Fakes handling a single HTTP request, returns a bad status"""
635
630
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
636
 
        self.wfile.write(b"Invalid status line\r\n")
 
631
        self.wfile.write("Invalid status line\r\n")
637
632
        # If we don't close the connection pycurl will hang. Since this is a
638
633
        # stress test we don't *have* to respect the protocol, but we don't
639
634
        # have to sabotage it too much either.
658
653
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
659
654
        # Returns an invalid protocol version, but curl just
660
655
        # ignores it and those cannot be tested.
661
 
        self.wfile.write(b"%s %d %s\r\n" % (
662
 
            b'HTTP/0.0', 404, b'Look at my protocol version'))
 
656
        self.wfile.write("%s %d %s\r\n" % ('HTTP/0.0',
 
657
                                           404,
 
658
                                           'Look at my protocol version'))
663
659
        return False
664
660
 
665
661
 
705
701
 
706
702
    def test_create(self):
707
703
        server = RecordingServer(expect_body_tail=None)
708
 
        self.assertEqual(b'', server.received_bytes)
 
704
        self.assertEqual('', server.received_bytes)
709
705
        self.assertEqual(None, server.host)
710
706
        self.assertEqual(None, server.port)
711
707
 
721
717
        self.assertEqual(None, server.port)
722
718
 
723
719
    def test_send_receive_bytes(self):
724
 
        server = RecordingServer(expect_body_tail=b'c', scheme='http')
 
720
        server = RecordingServer(expect_body_tail='c', scheme='http')
725
721
        self.start_server(server)
726
722
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
727
723
        sock.connect((server.host, server.port))
728
 
        sock.sendall(b'abc')
729
 
        self.assertEqual(b'HTTP/1.1 200 OK\r\n',
 
724
        sock.sendall('abc')
 
725
        self.assertEqual('HTTP/1.1 200 OK\r\n',
730
726
                         osutils.recv_all(sock, 4096))
731
 
        self.assertEqual(b'abc', server.received_bytes)
 
727
        self.assertEqual('abc', server.received_bytes)
732
728
 
733
729
 
734
730
class TestRangeRequestServer(TestSpecificRequestHandler):
744
740
    def test_readv(self):
745
741
        t = self.get_readonly_transport()
746
742
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
747
 
        self.assertEqual(l[0], (0, b'0'))
748
 
        self.assertEqual(l[1], (1, b'1'))
749
 
        self.assertEqual(l[2], (3, b'34'))
750
 
        self.assertEqual(l[3], (9, b'9'))
 
743
        self.assertEqual(l[0], (0, '0'))
 
744
        self.assertEqual(l[1], (1, '1'))
 
745
        self.assertEqual(l[2], (3, '34'))
 
746
        self.assertEqual(l[3], (9, '9'))
751
747
 
752
748
    def test_readv_out_of_order(self):
753
749
        t = self.get_readonly_transport()
754
750
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
755
 
        self.assertEqual(l[0], (1, b'1'))
756
 
        self.assertEqual(l[1], (9, b'9'))
757
 
        self.assertEqual(l[2], (0, b'0'))
758
 
        self.assertEqual(l[3], (3, b'34'))
 
751
        self.assertEqual(l[0], (1, '1'))
 
752
        self.assertEqual(l[1], (9, '9'))
 
753
        self.assertEqual(l[2], (0, '0'))
 
754
        self.assertEqual(l[3], (3, '34'))
759
755
 
760
756
    def test_readv_invalid_ranges(self):
761
757
        t = self.get_readonly_transport()
777
773
        t._max_readv_combine = 1
778
774
        t._max_get_ranges = 1
779
775
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
780
 
        self.assertEqual(l[0], (0, b'0'))
781
 
        self.assertEqual(l[1], (1, b'1'))
782
 
        self.assertEqual(l[2], (3, b'34'))
783
 
        self.assertEqual(l[3], (9, b'9'))
 
776
        self.assertEqual(l[0], (0, '0'))
 
777
        self.assertEqual(l[1], (1, '1'))
 
778
        self.assertEqual(l[2], (3, '34'))
 
779
        self.assertEqual(l[3], (9, '9'))
784
780
        # The server should have issued 4 requests
785
781
        self.assertEqual(4, server.GET_request_nb)
786
782
 
792
788
        # single range will keep its size even if bigger than the limit.
793
789
        t._get_max_size = 2
794
790
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
795
 
        self.assertEqual(l[0], (0, b'0'))
796
 
        self.assertEqual(l[1], (1, b'1'))
797
 
        self.assertEqual(l[2], (2, b'2345'))
798
 
        self.assertEqual(l[3], (6, b'6789'))
 
791
        self.assertEqual(l[0], (0, '0'))
 
792
        self.assertEqual(l[1], (1, '1'))
 
793
        self.assertEqual(l[2], (2, '2345'))
 
794
        self.assertEqual(l[3], (6, '6789'))
799
795
        # The server should have issued 3 requests
800
796
        self.assertEqual(3, server.GET_request_nb)
801
797
 
807
803
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
808
804
        # The server should have issued 3 requests
809
805
        self.assertEqual(3, server.GET_request_nb)
810
 
        self.assertEqual(b'0123456789', t.get_bytes('a'))
 
806
        self.assertEqual('0123456789', t.get_bytes('a'))
811
807
        self.assertEqual(4, server.GET_request_nb)
812
808
 
813
809
    def test_incomplete_readv_leave_pipe_clean(self):
818
814
        # Don't collapse readv results into a list so that we leave unread
819
815
        # bytes on the socket
820
816
        ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
821
 
        self.assertEqual((0, b'0'), next(ireadv))
 
817
        self.assertEqual((0, '0'), next(ireadv))
822
818
        # The server should have issued one request so far
823
819
        self.assertEqual(1, server.GET_request_nb)
824
 
        self.assertEqual(b'0123456789', t.get_bytes('a'))
 
820
        self.assertEqual('0123456789', t.get_bytes('a'))
825
821
        # get_bytes issued an additional request, the readv pending ones are
826
822
        # lost
827
823
        self.assertEqual(2, server.GET_request_nb)
881
877
 
882
878
 
883
879
class MultipleRangeWithoutContentLengthRequestHandler(
884
 
        http_server.TestingHTTPRequestHandler):
 
880
    http_server.TestingHTTPRequestHandler):
885
881
    """Reply to multiple range requests without content length header."""
886
882
 
887
883
    def get_multiple_ranges(self, file, file_size, ranges):
894
890
                         "multipart/byteranges; boundary=%s" % boundary)
895
891
        self.end_headers()
896
892
        for (start, end) in ranges:
897
 
            self.wfile.write(b"--%s\r\n" % boundary.encode('ascii'))
 
893
            self.wfile.write("--%s\r\n" % boundary)
898
894
            self.send_header("Content-type", 'application/octet-stream')
899
895
            self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
900
896
                                                                  end,
902
898
            self.end_headers()
903
899
            self.send_range_content(file, start, end - start + 1)
904
900
        # Final boundary
905
 
        self.wfile.write(b"--%s\r\n" % boundary)
 
901
        self.wfile.write("--%s\r\n" % boundary)
906
902
 
907
903
 
908
904
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
911
907
 
912
908
 
913
909
class TruncatedMultipleRangeRequestHandler(
914
 
        http_server.TestingHTTPRequestHandler):
 
910
    http_server.TestingHTTPRequestHandler):
915
911
    """Reply to multiple range requests truncating the last ones.
916
912
 
917
913
    This server generates responses whose Content-Length describes all the
927
923
        boundary = 'tagada'
928
924
        self.send_header('Content-Type',
929
925
                         'multipart/byteranges; boundary=%s' % boundary)
930
 
        boundary_line = b'--%s\r\n' % boundary.encode('ascii')
 
926
        boundary_line = '--%s\r\n' % boundary
931
927
        # Calculate the Content-Length
932
928
        content_length = 0
933
929
        for (start, end) in ranges:
936
932
                'Content-type', 'application/octet-stream')
937
933
            content_length += self._header_line_length(
938
934
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
939
 
            content_length += len('\r\n')  # end headers
940
 
            content_length += end - start  # + 1
 
935
            content_length += len('\r\n') # end headers
 
936
            content_length += end - start # + 1
941
937
        content_length += len(boundary_line)
942
938
        self.send_header('Content-length', content_length)
943
939
        self.end_headers()
974
970
        # Force separate ranges for each offset
975
971
        t._bytes_to_read_before_seek = 0
976
972
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
977
 
        self.assertEqual((0, b'0'), next(ireadv))
978
 
        self.assertEqual((2, b'2'), next(ireadv))
 
973
        self.assertEqual((0, '0'), next(ireadv))
 
974
        self.assertEqual((2, '2'), next(ireadv))
979
975
        # Only one request have been issued so far
980
976
        self.assertEqual(1, server.GET_request_nb)
981
 
        self.assertEqual((4, b'45'), next(ireadv))
982
 
        self.assertEqual((9, b'9'), next(ireadv))
 
977
        self.assertEqual((4, '45'), next(ireadv))
 
978
        self.assertEqual((9, '9'), next(ireadv))
983
979
        # We issue 3 requests: two multiple (4 ranges, then 2 ranges) then a
984
980
        # single range.
985
981
        self.assertEqual(3, server.GET_request_nb)
989
985
 
990
986
 
991
987
class TruncatedBeforeBoundaryRequestHandler(
992
 
        http_server.TestingHTTPRequestHandler):
 
988
    http_server.TestingHTTPRequestHandler):
993
989
    """Truncation before a boundary, like in bug 198646"""
994
990
 
995
991
    _truncated_ranges = 1
1000
996
        boundary = 'tagada'
1001
997
        self.send_header('Content-Type',
1002
998
                         'multipart/byteranges; boundary=%s' % boundary)
1003
 
        boundary_line = b'--%s\r\n' % boundary.encode('ascii')
 
999
        boundary_line = '--%s\r\n' % boundary
1004
1000
        # Calculate the Content-Length
1005
1001
        content_length = 0
1006
1002
        for (start, end) in ranges:
1009
1005
                'Content-type', 'application/octet-stream')
1010
1006
            content_length += self._header_line_length(
1011
1007
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
1012
 
            content_length += len('\r\n')  # end headers
1013
 
            content_length += end - start  # + 1
 
1008
            content_length += len('\r\n') # end headers
 
1009
            content_length += end - start # + 1
1014
1010
        content_length += len(boundary_line)
1015
1011
        self.send_header('Content-length', content_length)
1016
1012
        self.end_headers()
1048
1044
        # Force separate ranges for each offset
1049
1045
        t._bytes_to_read_before_seek = 0
1050
1046
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1051
 
        self.assertEqual((0, b'0'), next(ireadv))
1052
 
        self.assertEqual((2, b'2'), next(ireadv))
1053
 
        self.assertEqual((4, b'45'), next(ireadv))
1054
 
        self.assertEqual((9, b'9'), next(ireadv))
 
1047
        self.assertEqual((0, '0'), next(ireadv))
 
1048
        self.assertEqual((2, '2'), next(ireadv))
 
1049
        self.assertEqual((4, '45'), next(ireadv))
 
1050
        self.assertEqual((9, '9'), next(ireadv))
1055
1051
 
1056
1052
 
1057
1053
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1106
1102
    def test_few_ranges(self):
1107
1103
        t = self.get_readonly_transport()
1108
1104
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1109
 
        self.assertEqual(l[0], (0, b'0000'))
1110
 
        self.assertEqual(l[1], (1024, b'0001'))
 
1105
        self.assertEqual(l[0], (0, '0000'))
 
1106
        self.assertEqual(l[1], (1024, '0001'))
1111
1107
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1112
1108
 
1113
1109
    def test_more_ranges(self):
1114
1110
        t = self.get_readonly_transport()
1115
1111
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1116
 
        self.assertEqual(l[0], (0, b'0000'))
1117
 
        self.assertEqual(l[1], (1024, b'0001'))
1118
 
        self.assertEqual(l[2], (4096, b'0004'))
1119
 
        self.assertEqual(l[3], (8192, b'0008'))
 
1112
        self.assertEqual(l[0], (0, '0000'))
 
1113
        self.assertEqual(l[1], (1024, '0001'))
 
1114
        self.assertEqual(l[2], (4096, '0004'))
 
1115
        self.assertEqual(l[3], (8192, '0008'))
1120
1116
        # The server will refuse to serve the first request (too much ranges),
1121
1117
        # a second request will succeed.
1122
1118
        self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1129
1125
    """
1130
1126
 
1131
1127
    def _proxied_request(self):
1132
 
        handler = http.ProxyHandler()
1133
 
        request = http.Request('GET', 'http://baz/buzzle')
 
1128
        handler = _urllib2_wrappers.ProxyHandler()
 
1129
        request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
1134
1130
        handler.set_proxy(request, 'http')
1135
1131
        return request
1136
1132
 
1137
1133
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
1138
 
        handler = http.ProxyHandler()
 
1134
        handler = _urllib2_wrappers.ProxyHandler()
1139
1135
        self.assertEqual(expected,
1140
 
                         handler.evaluate_proxy_bypass(host, no_proxy))
 
1136
                          handler.evaluate_proxy_bypass(host, no_proxy))
1141
1137
 
1142
1138
    def test_empty_user(self):
1143
1139
        self.overrideEnv('http_proxy', 'http://bar.com')
1271
1267
 
1272
1268
    def _file_contents(self, relpath, ranges):
1273
1269
        t = self.get_readonly_transport()
1274
 
        offsets = [(start, end - start + 1) for start, end in ranges]
 
1270
        offsets = [ (start, end - start + 1) for start, end in ranges]
1275
1271
        coalesce = t._coalesce_offsets
1276
1272
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1277
1273
        code, data = t._get(relpath, coalesced)
1290
1286
    def test_range_header(self):
1291
1287
        # Valid ranges
1292
1288
        self.assertEqual(
1293
 
            [b'0', b'234'], list(self._file_contents('a', [(0, 0), (2, 4)])))
 
1289
            ['0', '234'], list(self._file_contents('a', [(0, 0), (2, 4)])))
1294
1290
 
1295
1291
    def test_range_header_tail(self):
1296
 
        self.assertEqual(b'789', self._file_tail('a', 3))
 
1292
        self.assertEqual('789', self._file_tail('a', 3))
1297
1293
 
1298
1294
    def test_syntactically_invalid_range_header(self):
1299
1295
        self.assertListRaises(errors.InvalidHttpRange,
1300
 
                              self._file_contents, 'a', [(4, 3)])
 
1296
                          self._file_contents, 'a', [(4, 3)])
1301
1297
 
1302
1298
    def test_semantically_invalid_range_header(self):
1303
1299
        self.assertListRaises(errors.InvalidHttpRange,
1304
 
                              self._file_contents, 'a', [(42, 128)])
 
1300
                          self._file_contents, 'a', [(42, 128)])
1305
1301
 
1306
1302
 
1307
1303
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1316
1312
        super(TestHTTPRedirections, self).setUp()
1317
1313
        self.build_tree_contents([('a', b'0123456789'),
1318
1314
                                  ('bundle',
1319
 
                                   b'# Bazaar revision bundle v0.9\n#\n')
 
1315
                                  b'# Bazaar revision bundle v0.9\n#\n')
1320
1316
                                  ],)
1321
1317
 
1322
1318
    def test_redirected(self):
1323
1319
        self.assertRaises(errors.RedirectRequested,
1324
1320
                          self.get_old_transport().get, 'a')
1325
 
        self.assertEqual(
1326
 
            b'0123456789',
1327
 
            self.get_new_transport().get('a').read())
1328
 
 
1329
 
 
1330
 
class RedirectedRequest(http.Request):
 
1321
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
 
1322
 
 
1323
 
 
1324
class RedirectedRequest(_urllib2_wrappers.Request):
1331
1325
    """Request following redirections. """
1332
1326
 
1333
 
    init_orig = http.Request.__init__
 
1327
    init_orig = _urllib2_wrappers.Request.__init__
1334
1328
 
1335
1329
    def __init__(self, method, url, *args, **kwargs):
1336
1330
        """Constructor.
1337
1331
 
1338
1332
        """
1339
1333
        # Since the tests using this class will replace
1340
 
        # http.Request, we can't just call the base class __init__
 
1334
        # _urllib2_wrappers.Request, we can't just call the base class __init__
1341
1335
        # or we'll loop.
1342
1336
        RedirectedRequest.init_orig(self, method, url, *args, **kwargs)
1343
1337
        self.follow_redirections = True
1344
1338
 
1345
1339
 
1346
1340
def install_redirected_request(test):
1347
 
    test.overrideAttr(http, 'Request', RedirectedRequest)
 
1341
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1348
1342
 
1349
1343
 
1350
1344
def cleanup_http_redirection_connections(test):
1351
1345
    # Some sockets are opened but never seen by _urllib, so we trap them at
1352
 
    # the http level to be able to clean them up.
 
1346
    # the _urllib2_wrappers level to be able to clean them up.
1353
1347
    def socket_disconnect(sock):
1354
1348
        try:
1355
1349
            sock.shutdown(socket.SHUT_RDWR)
1356
1350
            sock.close()
1357
1351
        except socket.error:
1358
1352
            pass
1359
 
 
1360
1353
    def connect(connection):
1361
1354
        test.http_connect_orig(connection)
1362
1355
        test.addCleanup(socket_disconnect, connection.sock)
1363
1356
    test.http_connect_orig = test.overrideAttr(
1364
 
        http.HTTPConnection, 'connect', connect)
1365
 
 
 
1357
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
1366
1358
    def connect(connection):
1367
1359
        test.https_connect_orig(connection)
1368
1360
        test.addCleanup(socket_disconnect, connection.sock)
1369
1361
    test.https_connect_orig = test.overrideAttr(
1370
 
        http.HTTPSConnection, 'connect', connect)
 
1362
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
1371
1363
 
1372
1364
 
1373
1365
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1375
1367
 
1376
1368
    http implementations do not redirect silently anymore (they
1377
1369
    do not redirect at all in fact). The mechanism is still in
1378
 
    place at the http.Request level and these tests
 
1370
    place at the _urllib2_wrappers.Request level and these tests
1379
1371
    exercise it.
1380
1372
    """
1381
1373
 
1403
1395
 
1404
1396
    def test_one_redirection(self):
1405
1397
        t = self.get_old_transport()
 
1398
        req = RedirectedRequest('GET', t._remote_path('a'))
1406
1399
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1407
1400
                                       self.new_server.port)
1408
1401
        self.old_server.redirections = \
1409
 
            [('(.*)', r'%s/1\1' % (new_prefix), 301), ]
1410
 
        self.assertEqual(
1411
 
            b'redirected once',
1412
 
            t.request('GET', t._remote_path('a'), retries=1).read())
 
1402
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
 
1403
        self.assertEqual('redirected once', t._perform(req).read())
1413
1404
 
1414
1405
    def test_five_redirections(self):
1415
1406
        t = self.get_old_transport()
 
1407
        req = RedirectedRequest('GET', t._remote_path('a'))
1416
1408
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1417
1409
                                       self.old_server.port)
1418
1410
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1424
1416
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1425
1417
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1426
1418
            ]
1427
 
        self.assertEqual(
1428
 
            b'redirected 5 times',
1429
 
            t.request('GET', t._remote_path('a'), retries=6).read())
 
1419
        self.assertEqual('redirected 5 times', t._perform(req).read())
1430
1420
 
1431
1421
 
1432
1422
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1439
1429
 
1440
1430
    def setUp(self):
1441
1431
        super(TestDoCatchRedirections, self).setUp()
1442
 
        self.build_tree_contents([('a', b'0123456789'), ],)
 
1432
        self.build_tree_contents([('a', b'0123456789'),],)
1443
1433
        cleanup_http_redirection_connections(self)
1444
1434
 
1445
1435
        self.old_transport = self.get_old_transport()
1451
1441
        t = self.get_new_transport()
1452
1442
 
1453
1443
        # We use None for redirected so that we fail if redirected
1454
 
        self.assertEqual(b'0123456789',
 
1444
        self.assertEqual('0123456789',
1455
1445
                         transport.do_catching_redirections(
1456
 
                             self.get_a, t, None).read())
 
1446
                self.get_a, t, None).read())
1457
1447
 
1458
1448
    def test_one_redirection(self):
1459
1449
        self.redirections = 0
1463
1453
            redirected_t = t._redirected_to(exception.source, exception.target)
1464
1454
            return redirected_t
1465
1455
 
1466
 
        self.assertEqual(b'0123456789',
 
1456
        self.assertEqual('0123456789',
1467
1457
                         transport.do_catching_redirections(
1468
 
                             self.get_a, self.old_transport, redirected).read())
 
1458
                self.get_a, self.old_transport, redirected).read())
1469
1459
        self.assertEqual(1, self.redirections)
1470
1460
 
1471
1461
    def test_redirection_loop(self):
1499
1489
        password = 'foo'
1500
1490
        _setup_authentication_config(scheme='http', host='localhost',
1501
1491
                                     user=user, password=password)
1502
 
        handler = http.HTTPAuthHandler()
 
1492
        handler = _urllib2_wrappers.HTTPAuthHandler()
1503
1493
        got_pass = handler.get_user_password(dict(
1504
1494
            user='joe',
1505
1495
            protocol='http',
1506
1496
            host='localhost',
1507
1497
            path='/',
1508
 
            realm=u'Realm',
 
1498
            realm='Realm',
1509
1499
            ))
1510
1500
        self.assertEqual((user, password), got_pass)
1511
1501
 
1523
1513
        super(TestAuth, self).setUp()
1524
1514
        self.server = self.get_readonly_server()
1525
1515
        self.build_tree_contents([('a', b'contents of a\n'),
1526
 
                                  ('b', b'contents of b\n'), ])
 
1516
                                  ('b', b'contents of b\n'),])
1527
1517
 
1528
1518
    def create_transport_readonly_server(self):
1529
1519
        server = self._auth_server(protocol_version=self._protocol_version)
1556
1546
    def test_empty_pass(self):
1557
1547
        self.server.add_user('joe', '')
1558
1548
        t = self.get_user_transport('joe', '')
1559
 
        self.assertEqual(b'contents of a\n', t.get('a').read())
 
1549
        self.assertEqual('contents of a\n', t.get('a').read())
1560
1550
        # Only one 'Authentication Required' error should occur
1561
1551
        self.assertEqual(1, self.server.auth_required_errors)
1562
1552
 
1563
1553
    def test_user_pass(self):
1564
1554
        self.server.add_user('joe', 'foo')
1565
1555
        t = self.get_user_transport('joe', 'foo')
1566
 
        self.assertEqual(b'contents of a\n', t.get('a').read())
 
1556
        self.assertEqual('contents of a\n', t.get('a').read())
1567
1557
        # Only one 'Authentication Required' error should occur
1568
1558
        self.assertEqual(1, self.server.auth_required_errors)
1569
1559
 
1589
1579
        t = self.get_user_transport(None, None)
1590
1580
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n')
1591
1581
        stdout, stderr = ui.ui_factory.stdout, ui.ui_factory.stderr
1592
 
        self.assertEqual(b'contents of a\n', t.get('a').read())
 
1582
        self.assertEqual('contents of a\n', t.get('a').read())
1593
1583
        # stdin should be empty
1594
1584
        self.assertEqual('', ui.ui_factory.stdin.readline())
1595
1585
        stderr.seek(0)
1604
1594
        t = self.get_user_transport('joe', None)
1605
1595
        ui.ui_factory = tests.TestUIFactory(stdin='foo\n')
1606
1596
        stdout, stderr = ui.ui_factory.stdout, ui.ui_factory.stderr
1607
 
        self.assertEqual(b'contents of a\n', t.get('a').read())
 
1597
        self.assertEqual('contents of a\n', t.get('a').read())
1608
1598
        # stdin should be empty
1609
1599
        self.assertEqual('', ui.ui_factory.stdin.readline())
1610
1600
        self._check_password_prompt(t._unqualified_scheme, 'joe',
1612
1602
        self.assertEqual('', stdout.getvalue())
1613
1603
        # And we shouldn't prompt again for a different request
1614
1604
        # against the same transport.
1615
 
        self.assertEqual(b'contents of b\n', t.get('b').read())
 
1605
        self.assertEqual('contents of b\n', t.get('b').read())
1616
1606
        t2 = t.clone()
1617
1607
        # And neither against a clone
1618
 
        self.assertEqual(b'contents of b\n', t2.get('b').read())
 
1608
        self.assertEqual('contents of b\n', t2.get('b').read())
1619
1609
        # Only one 'Authentication Required' error should occur
1620
1610
        self.assertEqual(1, self.server.auth_required_errors)
1621
1611
 
1630
1620
    def _expected_username_prompt(self, scheme):
1631
1621
        return (self._username_prompt_prefix
1632
1622
                + "%s %s:%d, Realm: '%s' username: " % (scheme.upper(),
1633
 
                                                        self.server.host, self.server.port,
1634
 
                                                        self.server.auth_realm))
 
1623
                                 self.server.host, self.server.port,
 
1624
                                 self.server.auth_realm))
1635
1625
 
1636
1626
    def test_no_prompt_for_password_when_using_auth_config(self):
1637
 
        user = ' joe'
 
1627
        user =' joe'
1638
1628
        password = 'foo'
1639
1629
        stdin_content = 'bar\n'  # Not the right password
1640
1630
        self.server.add_user(user, password)
1644
1634
        _setup_authentication_config(scheme='http', port=self.server.port,
1645
1635
                                     user=user, password=password)
1646
1636
        # Issue a request to the server to connect
1647
 
        with t.get('a') as f:
1648
 
            self.assertEqual(b'contents of a\n', f.read())
 
1637
        self.assertEqual('contents of a\n', t.get('a').read())
1649
1638
        # stdin should have  been left untouched
1650
1639
        self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1651
1640
        # Only one 'Authentication Required' error should occur
1657
1646
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1658
1647
        self.server.add_user('joe', 'foo')
1659
1648
        t = self.get_user_transport('joe', 'foo')
1660
 
        with t.get('a') as f:
1661
 
            self.assertEqual(b'contents of a\n', f.read())
1662
 
        with t.get('b') as f:
1663
 
            self.assertEqual(b'contents of b\n', f.read())
 
1649
        self.assertEqual('contents of a\n', t.get('a').read())
 
1650
        self.assertEqual('contents of b\n', t.get('b').read())
1664
1651
        # Only one 'Authentication Required' error should have
1665
1652
        # occured so far
1666
1653
        self.assertEqual(1, self.server.auth_required_errors)
1667
1654
        # The server invalidates the current nonce
1668
1655
        self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1669
 
        self.assertEqual(b'contents of a\n', t.get('a').read())
 
1656
        self.assertEqual('contents of a\n', t.get('a').read())
1670
1657
        # Two 'Authentication Required' errors should occur (the
1671
1658
        # initial 'who are you' and a second 'who are you' with the new nonce)
1672
1659
        self.assertEqual(2, self.server.auth_required_errors)
1679
1666
                                     user=user, password=password)
1680
1667
        t = self.get_user_transport(None, None)
1681
1668
        # Issue a request to the server to connect
1682
 
        with t.get('a') as f:
1683
 
            self.assertEqual(b'contents of a\n', f.read())
 
1669
        self.assertEqual('contents of a\n', t.get('a').read())
1684
1670
        # Only one 'Authentication Required' error should occur
1685
1671
        self.assertEqual(1, self.server.auth_required_errors)
1686
1672
 
1692
1678
        t = self.get_user_transport(user, password)
1693
1679
        # Capture the debug calls to mutter
1694
1680
        self.mutters = []
1695
 
 
1696
1681
        def mutter(*args):
1697
1682
            lines = args[0] % args[1:]
1698
1683
            # Some calls output multiple lines, just split them now since we
1828
1813
        httpd = self.http_server.server
1829
1814
 
1830
1815
        socket = SampleSocket(
1831
 
            b'POST /.bzr/smart %s \r\n' % self._protocol_version.encode('ascii') +
 
1816
            b'POST /.bzr/smart %s \r\n' % self._protocol_version.encode('ascii')
1832
1817
            # HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1833
1818
            # for 1.0)
1834
 
            b'Content-Length: 6\r\n'
 
1819
            + b'Content-Length: 6\r\n'
1835
1820
            b'\r\n'
1836
1821
            b'hello\n')
1837
1822
        # Beware: the ('localhost', 80) below is the
1844
1829
                                                         httpd)
1845
1830
        response = socket.writefile.getvalue()
1846
1831
        self.assertStartsWith(
1847
 
            response,
1848
 
            b'%s 200 ' % self._protocol_version.encode('ascii'))
 
1832
                response,
 
1833
                b'%s 200 ' % self._protocol_version.encode('ascii'))
1849
1834
        # This includes the end of the HTTP headers, and all the body.
1850
1835
        expected_end_of_response = b'\r\n\r\nok\x012\n'
1851
1836
        self.assertEndsWith(response, expected_end_of_response)
1902
1887
                             'http://foo.example.com/foo/subdir')
1903
1888
        self.assertIsInstance(r, type(t))
1904
1889
        self.assertEqual('http://foo.example.com/foo/subdir/',
1905
 
                         r.external_url())
 
1890
            r.external_url())
1906
1891
 
1907
1892
    def test_redirected_to_same_host_sibling_protocol(self):
1908
1893
        t = self._transport('http://www.example.com/foo')
1910
1895
                             'https://www.example.com/foo')
1911
1896
        self.assertIsInstance(r, type(t))
1912
1897
        self.assertEqual('https://www.example.com/foo/',
1913
 
                         r.external_url())
 
1898
            r.external_url())
1914
1899
 
1915
1900
    def test_redirected_to_same_host_different_protocol(self):
1916
1901
        t = self._transport('http://www.example.com/foo')
1949
1934
    def _handle_one_request(self):
1950
1935
        tcs = self.server.test_case_server
1951
1936
        requestline = self.rfile.readline()
1952
 
        if PY3:
1953
 
            headers = parse_headers(self.rfile)
1954
 
            bytes_read = len(headers.as_bytes())
1955
 
            bytes_read += headers.as_bytes().count(b'\n')
1956
 
            bytes_read += len(requestline)
1957
 
        else:
1958
 
            headers = self.MessageClass(self.rfile, 0)
1959
 
            # We just read: the request, the headers, an empty line indicating the
1960
 
            # end of the headers.
1961
 
            bytes_read = len(requestline)
1962
 
            for line in headers.headers:
1963
 
                bytes_read += len(line)
1964
 
            bytes_read += len(b'\r\n')
1965
 
        if requestline.startswith(b'POST'):
 
1937
        headers = self.MessageClass(self.rfile, 0)
 
1938
        # We just read: the request, the headers, an empty line indicating the
 
1939
        # end of the headers.
 
1940
        bytes_read = len(requestline)
 
1941
        for line in headers.headers:
 
1942
            bytes_read += len(line)
 
1943
        bytes_read += len(b'\r\n')
 
1944
        if requestline.startswith('POST'):
1966
1945
            # The body should be a single line (or we don't know where it ends
1967
1946
            # and we don't want to issue a blocking read)
1968
1947
            body = self.rfile.readline()
1998
1977
 
1999
1978
if features.HTTPSServerFeature.available():
2000
1979
    from . import https_server
2001
 
 
2002
1980
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
2003
1981
        pass
2004
1982
 
2014
1992
        self.server = self._activity_server(self._protocol_version)
2015
1993
        self.server.start_server()
2016
1994
        self.addCleanup(self.server.stop_server)
2017
 
        _activities = {}  # Don't close over self and create a cycle
2018
 
 
 
1995
        _activities = {} # Don't close over self and create a cycle
2019
1996
        def report_activity(t, bytes, direction):
2020
1997
            count = _activities.get(direction, 0)
2021
1998
            count += bytes
2051
2028
Bazaar-NG meta directory, format 1
2052
2029
'''
2053
2030
        t = self.get_transport()
2054
 
        self.assertEqual(b'Bazaar-NG meta directory, format 1\n',
 
2031
        self.assertEqual('Bazaar-NG meta directory, format 1\n',
2055
2032
                         t.get('foo/bar').read())
2056
2033
        self.assertActivitiesMatch()
2057
2034
 
2122
2099
        # Remember that the request is ignored and that the ranges below
2123
2100
        # doesn't have to match the canned response.
2124
2101
        l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2125
 
        # Force consumption of the last bytesrange boundary
2126
 
        t._get_connection().cleanup_pipe()
2127
2102
        self.assertEqual(2, len(l))
2128
2103
        self.assertActivitiesMatch()
2129
2104
 
2172
2147
 
2173
2148
    def setUp(self):
2174
2149
        super(TestNoReportActivity, self).setUp()
2175
 
        self._transport = HttpTransport
 
2150
        self._transport =HttpTransport
2176
2151
        TestActivityMixin.setUp(self)
2177
2152
 
2178
2153
    def assertActivitiesMatch(self):
2200
2175
        new_prefix = 'http://%s:%s' % (self.new_server.host,
2201
2176
                                       self.new_server.port)
2202
2177
        self.old_server.redirections = [
2203
 
            ('(.*)', r'%s/1\1' % (new_prefix), 301), ]
 
2178
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2204
2179
        self.old_transport = self.get_old_transport()
2205
2180
        self.new_server.add_user('joe', 'foo')
2206
2181
        cleanup_http_redirection_connections(self)
2223
2198
            return redirected_t
2224
2199
 
2225
2200
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n')
2226
 
        self.assertEqual(b'redirected once',
 
2201
        self.assertEqual('redirected once',
2227
2202
                         transport.do_catching_redirections(
2228
 
                             self.get_a, self.old_transport, redirected).read())
 
2203
                self.get_a, self.old_transport, redirected).read())
2229
2204
        self.assertEqual(1, self.redirections)
2230
2205
        # stdin should be empty
2231
2206
        self.assertEqual('', ui.ui_factory.stdin.readline())
2236
2211
        self.new_server.add_user('joe', 'foo')
2237
2212
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n')
2238
2213
        t = self.old_transport
 
2214
        req = RedirectedRequest('GET', t.abspath('a'))
2239
2215
        new_prefix = 'http://%s:%s' % (self.new_server.host,
2240
2216
                                       self.new_server.port)
2241
2217
        self.old_server.redirections = [
2242
 
            ('(.*)', r'%s/1\1' % (new_prefix), 301), ]
2243
 
        self.assertEqual(
2244
 
            b'redirected once',
2245
 
            t.request('GET', t.abspath('a'), retries=3).read())
 
2218
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
 
2219
        self.assertEqual('redirected once', t._perform(req).read())
2246
2220
        # stdin should be empty
2247
2221
        self.assertEqual('', ui.ui_factory.stdin.readline())
2248
2222
        # stdout should be empty, stderr will contains the prompts
2249
2223
        self.assertEqual('', ui.ui_factory.stdout.getvalue())
 
2224