/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to breezy/tests/test_http.py

  • Committer: Jelmer Vernooij
  • Date: 2017-05-22 00:56:52 UTC
  • mfrom: (6621.2.26 py3_pokes)
  • Revision ID: jelmer@jelmer.uk-20170522005652-yjahcr9hwmjkno7n
Merge Python3 porting work ('py3 pokes')

Show diffs side-by-side

added added

removed removed

Lines of Context:
breezy/tests/test_http.py
1
 
# Copyright (C) 2005-2010 Canonical Ltd
 
1
# Copyright (C) 2005-2012, 2015, 2016, 2017 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
20
20
transport implementation, http protocol versions and authentication schemes.
21
21
"""
22
22
 
23
 
# TODO: Should be renamed to bzrlib.transport.http.tests?
24
 
# TODO: What about renaming to bzrlib.tests.transport.http ?
 
23
# TODO: Should be renamed to breezy.transport.http.tests?
 
24
# TODO: What about renaming to breezy.tests.transport.http ?
25
25
 
26
 
from cStringIO import StringIO
27
26
import httplib
28
 
import os
29
 
import select
 
27
import io
30
28
import SimpleHTTPServer
31
29
import socket
32
30
import sys
33
31
import threading
34
32
 
35
 
import bzrlib
36
 
from bzrlib import (
37
 
    bzrdir,
 
33
import breezy
 
34
from .. import (
38
35
    config,
 
36
    controldir,
 
37
    debug,
39
38
    errors,
40
39
    osutils,
41
40
    remote as _mod_remote,
42
41
    tests,
 
42
    trace,
43
43
    transport,
44
44
    ui,
45
 
    urlutils,
46
 
    )
47
 
from bzrlib.symbol_versioning import (
48
 
    deprecated_in,
49
 
    )
50
 
from bzrlib.tests import (
 
45
    )
 
46
from . import (
51
47
    features,
52
48
    http_server,
53
49
    http_utils,
54
 
    )
55
 
from bzrlib.transport import (
 
50
    test_server,
 
51
    )
 
52
from .scenarios import (
 
53
    load_tests_apply_scenarios,
 
54
    multiply_scenarios,
 
55
    )
 
56
from ..transport import (
56
57
    http,
57
58
    remote,
58
59
    )
59
 
from bzrlib.transport.http import (
 
60
from ..transport.http import (
60
61
    _urllib,
61
62
    _urllib2_wrappers,
62
63
    )
63
64
 
64
65
 
65
66
if features.pycurl.available():
66
 
    from bzrlib.transport.http._pycurl import PyCurlTransport
67
 
 
68
 
 
69
 
def load_tests(standard_tests, module, loader):
70
 
    """Multiply tests for http clients and protocol versions."""
71
 
    result = loader.suiteClass()
72
 
 
73
 
    # one for each transport implementation
74
 
    t_tests, remaining_tests = tests.split_suite_by_condition(
75
 
        standard_tests, tests.condition_isinstance((
76
 
                TestHttpTransportRegistration,
77
 
                TestHttpTransportUrls,
78
 
                Test_redirected_to,
79
 
                )))
 
67
    from ..transport.http._pycurl import PyCurlTransport
 
68
 
 
69
 
 
70
load_tests = load_tests_apply_scenarios
 
71
 
 
72
 
 
73
def vary_by_http_client_implementation():
 
74
    """Test the two libraries we can use, pycurl and urllib."""
80
75
    transport_scenarios = [
81
76
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
82
77
                        _server=http_server.HttpServer_urllib,
83
 
                        _qualified_prefix='http+urllib',)),
 
78
                        _url_protocol='http+urllib',)),
84
79
        ]
85
80
    if features.pycurl.available():
86
81
        transport_scenarios.append(
87
82
            ('pycurl', dict(_transport=PyCurlTransport,
88
83
                            _server=http_server.HttpServer_PyCurl,
89
 
                            _qualified_prefix='http+pycurl',)))
90
 
    tests.multiply_tests(t_tests, transport_scenarios, result)
91
 
 
92
 
    # each implementation tested with each HTTP version
93
 
    tp_tests, remaining_tests = tests.split_suite_by_condition(
94
 
        remaining_tests, tests.condition_isinstance((
95
 
                SmartHTTPTunnellingTest,
96
 
                TestDoCatchRedirections,
97
 
                TestHTTPConnections,
98
 
                TestHTTPRedirections,
99
 
                TestHTTPSilentRedirections,
100
 
                TestLimitedRangeRequestServer,
101
 
                TestPost,
102
 
                TestProxyHttpServer,
103
 
                TestRanges,
104
 
                TestSpecificRequestHandler,
105
 
                )))
106
 
    protocol_scenarios = [
107
 
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
108
 
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
109
 
            ]
110
 
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
111
 
                                            protocol_scenarios)
112
 
    tests.multiply_tests(tp_tests, tp_scenarios, result)
113
 
 
114
 
    # proxy auth: each auth scheme on all http versions on all implementations.
115
 
    tppa_tests, remaining_tests = tests.split_suite_by_condition(
116
 
        remaining_tests, tests.condition_isinstance((
117
 
                TestProxyAuth,
118
 
                )))
119
 
    proxy_auth_scheme_scenarios = [
120
 
        ('basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
121
 
        ('digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
122
 
        ('basicdigest',
123
 
         dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
84
                            _url_protocol='http+pycurl',)))
 
85
    return transport_scenarios
 
86
 
 
87
 
 
88
def vary_by_http_protocol_version():
 
89
    """Test on http/1.0 and 1.1"""
 
90
    return [
 
91
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
92
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
124
93
        ]
125
 
    tppa_scenarios = tests.multiply_scenarios(tp_scenarios,
126
 
                                              proxy_auth_scheme_scenarios)
127
 
    tests.multiply_tests(tppa_tests, tppa_scenarios, result)
128
 
 
129
 
    # auth: each auth scheme on all http versions on all implementations.
130
 
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
131
 
        remaining_tests, tests.condition_isinstance((
132
 
                TestAuth,
133
 
                )))
134
 
    auth_scheme_scenarios = [
 
94
 
 
95
 
 
96
def vary_by_http_auth_scheme():
 
97
    scenarios = [
135
98
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
136
99
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
137
100
        ('basicdigest',
138
 
         dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
139
 
        ]
140
 
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
141
 
                                             auth_scheme_scenarios)
142
 
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
143
 
 
144
 
    # activity: on all http[s] versions on all implementations
145
 
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
146
 
        remaining_tests, tests.condition_isinstance((
147
 
                TestActivity,
148
 
                )))
 
101
            dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
 
102
        ]
 
103
    # Add some attributes common to all scenarios
 
104
    for scenario_id, scenario_dict in scenarios:
 
105
        scenario_dict.update(_auth_header='Authorization',
 
106
                             _username_prompt_prefix='',
 
107
                             _password_prompt_prefix='')
 
108
    return scenarios
 
109
 
 
110
 
 
111
def vary_by_http_proxy_auth_scheme():
 
112
    scenarios = [
 
113
        ('proxy-basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
 
114
        ('proxy-digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
 
115
        ('proxy-basicdigest',
 
116
            dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
117
        ]
 
118
    # Add some attributes common to all scenarios
 
119
    for scenario_id, scenario_dict in scenarios:
 
120
        scenario_dict.update(_auth_header='Proxy-Authorization',
 
121
                             _username_prompt_prefix='Proxy ',
 
122
                             _password_prompt_prefix='Proxy ')
 
123
    return scenarios
 
124
 
 
125
 
 
126
def vary_by_http_activity():
149
127
    activity_scenarios = [
150
128
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
151
 
                             _transport=_urllib.HttpTransport_urllib,)),
 
129
                            _transport=_urllib.HttpTransport_urllib,)),
152
130
        ]
153
 
    if tests.HTTPSServerFeature.available():
 
131
    if features.pycurl.available():
 
132
        activity_scenarios.append(
 
133
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
 
134
                                _transport=PyCurlTransport,)),)
 
135
    if features.HTTPSServerFeature.available():
 
136
        # FIXME: Until we have a better way to handle self-signed certificates
 
137
        # (like allowing them in a test specific authentication.conf for
 
138
        # example), we need some specialized pycurl/urllib transport for tests.
 
139
        # -- vila 2012-01-20
 
140
        from . import (
 
141
            ssl_certs,
 
142
            )
 
143
        class HTTPS_urllib_transport(_urllib.HttpTransport_urllib):
 
144
 
 
145
            def __init__(self, base, _from_transport=None):
 
146
                super(HTTPS_urllib_transport, self).__init__(
 
147
                    base, _from_transport=_from_transport,
 
148
                    ca_certs=ssl_certs.build_path('ca.crt'))
 
149
 
154
150
        activity_scenarios.append(
155
151
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
156
 
                                  _transport=_urllib.HttpTransport_urllib,)),)
157
 
    if features.pycurl.available():
158
 
        activity_scenarios.append(
159
 
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
160
 
                                 _transport=PyCurlTransport,)),)
161
 
        if tests.HTTPSServerFeature.available():
162
 
            from bzrlib.tests import (
163
 
                ssl_certs,
164
 
                )
165
 
            # FIXME: Until we have a better way to handle self-signed
166
 
            # certificates (like allowing them in a test specific
167
 
            # authentication.conf for example), we need some specialized pycurl
168
 
            # transport for tests.
 
152
                                  _transport=HTTPS_urllib_transport,)),)
 
153
        if features.pycurl.available():
169
154
            class HTTPS_pycurl_transport(PyCurlTransport):
170
155
 
171
156
                def __init__(self, base, _from_transport=None):
175
160
 
176
161
            activity_scenarios.append(
177
162
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
178
 
                                      _transport=HTTPS_pycurl_transport,)),)
179
 
 
180
 
    tpact_scenarios = tests.multiply_scenarios(activity_scenarios,
181
 
                                               protocol_scenarios)
182
 
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
183
 
 
184
 
    # No parametrization for the remaining tests
185
 
    result.addTests(remaining_tests)
186
 
 
187
 
    return result
 
163
                                    _transport=HTTPS_pycurl_transport,)),)
 
164
    return activity_scenarios
188
165
 
189
166
 
190
167
class FakeManager(object):
223
200
        self._sock.bind(('127.0.0.1', 0))
224
201
        self.host, self.port = self._sock.getsockname()
225
202
        self._ready = threading.Event()
226
 
        self._thread = threading.Thread(target=self._accept_read_and_reply)
227
 
        self._thread.setDaemon(True)
 
203
        self._thread = test_server.TestThread(
 
204
            sync_event=self._ready, target=self._accept_read_and_reply)
228
205
        self._thread.start()
229
 
        self._ready.wait(5)
 
206
        if 'threads' in tests.selftest_debug_flags:
 
207
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
 
208
        self._ready.wait()
230
209
 
231
210
    def _accept_read_and_reply(self):
232
211
        self._sock.listen(1)
233
212
        self._ready.set()
234
 
        self._sock.settimeout(5)
235
 
        try:
236
 
            conn, address = self._sock.accept()
237
 
            # On win32, the accepted connection will be non-blocking to start
238
 
            # with because we're using settimeout.
239
 
            conn.setblocking(True)
 
213
        conn, address = self._sock.accept()
 
214
        if self._expect_body_tail is not None:
240
215
            while not self.received_bytes.endswith(self._expect_body_tail):
241
216
                self.received_bytes += conn.recv(4096)
242
217
            conn.sendall('HTTP/1.1 200 OK\r\n')
243
 
        except socket.timeout:
244
 
            # Make sure the client isn't stuck waiting for us to e.g. accept.
 
218
        try:
245
219
            self._sock.close()
246
220
        except socket.error:
247
221
            # The client may have already closed the socket.
249
223
 
250
224
    def stop_server(self):
251
225
        try:
252
 
            self._sock.close()
 
226
            # Issue a fake connection to wake up the server and allow it to
 
227
            # finish quickly
 
228
            fake_conn = osutils.connect_socket((self.host, self.port))
 
229
            fake_conn.close()
253
230
        except socket.error:
254
231
            # We might have already closed it.  We don't care.
255
232
            pass
256
233
        self.host = None
257
234
        self.port = None
 
235
        self._thread.join()
 
236
        if 'threads' in tests.selftest_debug_flags:
 
237
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
258
238
 
259
239
 
260
240
class TestAuthHeader(tests.TestCase):
281
261
        self.assertEqual('basic', scheme)
282
262
        self.assertEqual('realm="Thou should not pass"', remainder)
283
263
 
 
264
    def test_build_basic_header_with_long_creds(self):
 
265
        handler = _urllib2_wrappers.BasicAuthHandler()
 
266
        user = 'user' * 10  # length 40
 
267
        password = 'password' * 5  # length 40
 
268
        header = handler.build_auth_header(
 
269
            dict(user=user, password=password), None)
 
270
        # https://bugs.launchpad.net/bzr/+bug/1606203 was caused by incorrectly
 
271
        # creating a header value with an embedded '\n'
 
272
        self.assertFalse('\n' in header)
 
273
 
284
274
    def test_basic_extract_realm(self):
285
275
        scheme, remainder = self.parse_header(
286
276
            'Basic realm="Thou should not pass"',
296
286
        self.assertEqual('realm="Thou should not pass"', remainder)
297
287
 
298
288
 
 
289
class TestHTTPRangeParsing(tests.TestCase):
 
290
 
 
291
    def setUp(self):
 
292
        super(TestHTTPRangeParsing, self).setUp()
 
293
        # We focus on range  parsing here and ignore everything else
 
294
        class RequestHandler(http_server.TestingHTTPRequestHandler):
 
295
            def setup(self): pass
 
296
            def handle(self): pass
 
297
            def finish(self): pass
 
298
 
 
299
        self.req_handler = RequestHandler(None, None, None)
 
300
 
 
301
    def assertRanges(self, ranges, header, file_size):
 
302
        self.assertEqual(ranges,
 
303
                          self.req_handler._parse_ranges(header, file_size))
 
304
 
 
305
    def test_simple_range(self):
 
306
        self.assertRanges([(0,2)], 'bytes=0-2', 12)
 
307
 
 
308
    def test_tail(self):
 
309
        self.assertRanges([(8, 11)], 'bytes=-4', 12)
 
310
 
 
311
    def test_tail_bigger_than_file(self):
 
312
        self.assertRanges([(0, 11)], 'bytes=-99', 12)
 
313
 
 
314
    def test_range_without_end(self):
 
315
        self.assertRanges([(4, 11)], 'bytes=4-', 12)
 
316
 
 
317
    def test_invalid_ranges(self):
 
318
        self.assertRanges(None, 'bytes=12-22', 12)
 
319
        self.assertRanges(None, 'bytes=1-3,12-22', 12)
 
320
        self.assertRanges(None, 'bytes=-', 12)
 
321
 
 
322
 
299
323
class TestHTTPServer(tests.TestCase):
300
324
    """Test the HTTP servers implementations."""
301
325
 
304
328
 
305
329
            protocol_version = 'HTTP/0.1'
306
330
 
307
 
        server = http_server.HttpServer(BogusRequestHandler)
308
 
        try:
309
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
310
 
        except:
311
 
            server.stop_server()
312
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
331
        self.assertRaises(httplib.UnknownProtocol,
 
332
                          http_server.HttpServer, BogusRequestHandler)
313
333
 
314
334
    def test_force_invalid_protocol(self):
315
 
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
316
 
        try:
317
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
318
 
        except:
319
 
            server.stop_server()
320
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
335
        self.assertRaises(httplib.UnknownProtocol,
 
336
                          http_server.HttpServer, protocol_version='HTTP/0.1')
321
337
 
322
338
    def test_server_start_and_stop(self):
323
339
        server = http_server.HttpServer()
 
340
        self.addCleanup(server.stop_server)
324
341
        server.start_server()
325
 
        try:
326
 
            self.assertTrue(server._http_running)
327
 
        finally:
328
 
            server.stop_server()
329
 
        self.assertFalse(server._http_running)
 
342
        self.assertTrue(server.server is not None)
 
343
        self.assertTrue(server.server.serving is not None)
 
344
        self.assertTrue(server.server.serving)
330
345
 
331
346
    def test_create_http_server_one_zero(self):
332
347
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
335
350
 
336
351
        server = http_server.HttpServer(RequestHandlerOneZero)
337
352
        self.start_server(server)
338
 
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
 
353
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
339
354
 
340
355
    def test_create_http_server_one_one(self):
341
356
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
344
359
 
345
360
        server = http_server.HttpServer(RequestHandlerOneOne)
346
361
        self.start_server(server)
347
 
        self.assertIsInstance(server._httpd,
 
362
        self.assertIsInstance(server.server,
348
363
                              http_server.TestingThreadingHTTPServer)
349
364
 
350
365
    def test_create_http_server_force_one_one(self):
355
370
        server = http_server.HttpServer(RequestHandlerOneZero,
356
371
                                        protocol_version='HTTP/1.1')
357
372
        self.start_server(server)
358
 
        self.assertIsInstance(server._httpd,
 
373
        self.assertIsInstance(server.server,
359
374
                              http_server.TestingThreadingHTTPServer)
360
375
 
361
376
    def test_create_http_server_force_one_zero(self):
366
381
        server = http_server.HttpServer(RequestHandlerOneOne,
367
382
                                        protocol_version='HTTP/1.0')
368
383
        self.start_server(server)
369
 
        self.assertIsInstance(server._httpd,
 
384
        self.assertIsInstance(server.server,
370
385
                              http_server.TestingHTTPServer)
371
386
 
372
387
 
380
395
    _transport = property(_get_pycurl_maybe)
381
396
 
382
397
 
383
 
class TestHttpUrls(tests.TestCase):
384
 
 
385
 
    # TODO: This should be moved to authorization tests once they
386
 
    # are written.
387
 
 
388
 
    def test_url_parsing(self):
389
 
        f = FakeManager()
390
 
        url = http.extract_auth('http://example.com', f)
391
 
        self.assertEqual('http://example.com', url)
392
 
        self.assertEqual(0, len(f.credentials))
393
 
        url = http.extract_auth(
394
 
            'http://user:pass@example.com/bzr/bzr.dev', f)
395
 
        self.assertEqual('http://example.com/bzr/bzr.dev', url)
396
 
        self.assertEqual(1, len(f.credentials))
397
 
        self.assertEqual([None, 'example.com', 'user', 'pass'],
398
 
                         f.credentials[0])
399
 
 
400
 
 
401
398
class TestHttpTransportUrls(tests.TestCase):
402
399
    """Test the http urls."""
403
400
 
 
401
    scenarios = vary_by_http_client_implementation()
 
402
 
404
403
    def test_abs_url(self):
405
404
        """Construction of absolute http URLs"""
406
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
405
        t = self._transport('http://example.com/bzr/bzr.dev/')
407
406
        eq = self.assertEqualDiff
408
 
        eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
409
 
        eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
410
 
        eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
 
407
        eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
 
408
        eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
 
409
        eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
411
410
        eq(t.abspath('.bzr/1//2/./3'),
412
 
           'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
 
411
           'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
413
412
 
414
413
    def test_invalid_http_urls(self):
415
414
        """Trap invalid construction of urls"""
416
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
415
        self._transport('http://example.com/bzr/bzr.dev/')
417
416
        self.assertRaises(errors.InvalidURL,
418
417
                          self._transport,
419
 
                          'http://http://bazaar-vcs.org/bzr/bzr.dev/')
 
418
                          'http://http://example.com/bzr/bzr.dev/')
420
419
 
421
420
    def test_http_root_urls(self):
422
421
        """Construction of URLs from server root"""
423
 
        t = self._transport('http://bzr.ozlabs.org/')
 
422
        t = self._transport('http://example.com/')
424
423
        eq = self.assertEqualDiff
425
424
        eq(t.abspath('.bzr/tree-version'),
426
 
           'http://bzr.ozlabs.org/.bzr/tree-version')
 
425
           'http://example.com/.bzr/tree-version')
427
426
 
428
427
    def test_http_impl_urls(self):
429
428
        """There are servers which ask for particular clients to connect"""
431
430
        server.start_server()
432
431
        try:
433
432
            url = server.get_url()
434
 
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
 
433
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
435
434
        finally:
436
435
            server.stop_server()
437
436
 
475
474
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
476
475
    """Test the http connections."""
477
476
 
 
477
    scenarios = multiply_scenarios(
 
478
        vary_by_http_client_implementation(),
 
479
        vary_by_http_protocol_version(),
 
480
        )
 
481
 
478
482
    def setUp(self):
479
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
483
        super(TestHTTPConnections, self).setUp()
480
484
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
481
485
                        transport=self.get_transport())
482
486
 
483
487
    def test_http_has(self):
484
488
        server = self.get_readonly_server()
485
 
        t = self._transport(server.get_url())
 
489
        t = self.get_readonly_transport()
486
490
        self.assertEqual(t.has('foo/bar'), True)
487
491
        self.assertEqual(len(server.logs), 1)
488
492
        self.assertContainsRe(server.logs[0],
490
494
 
491
495
    def test_http_has_not_found(self):
492
496
        server = self.get_readonly_server()
493
 
        t = self._transport(server.get_url())
 
497
        t = self.get_readonly_transport()
494
498
        self.assertEqual(t.has('not-found'), False)
495
499
        self.assertContainsRe(server.logs[1],
496
500
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
497
501
 
498
502
    def test_http_get(self):
499
503
        server = self.get_readonly_server()
500
 
        t = self._transport(server.get_url())
 
504
        t = self.get_readonly_transport()
501
505
        fp = t.get('foo/bar')
502
506
        self.assertEqualDiff(
503
507
            fp.read(),
505
509
        self.assertEqual(len(server.logs), 1)
506
510
        self.assertTrue(server.logs[0].find(
507
511
            '"GET /foo/bar HTTP/1.1" 200 - "-" "bzr/%s'
508
 
            % bzrlib.__version__) > -1)
 
512
            % breezy.__version__) > -1)
509
513
 
510
514
    def test_has_on_bogus_host(self):
511
515
        # Get a free address and don't 'accept' on it, so that we
525
529
class TestHttpTransportRegistration(tests.TestCase):
526
530
    """Test registrations of various http implementations"""
527
531
 
 
532
    scenarios = vary_by_http_client_implementation()
 
533
 
528
534
    def test_http_registered(self):
529
 
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
 
535
        t = transport.get_transport_from_url(
 
536
            '%s://foo.com/' % self._url_protocol)
530
537
        self.assertIsInstance(t, transport.Transport)
531
538
        self.assertIsInstance(t, self._transport)
532
539
 
533
540
 
534
541
class TestPost(tests.TestCase):
535
542
 
 
543
    scenarios = multiply_scenarios(
 
544
        vary_by_http_client_implementation(),
 
545
        vary_by_http_protocol_version(),
 
546
        )
 
547
 
536
548
    def test_post_body_is_received(self):
537
549
        server = RecordingServer(expect_body_tail='end-of-body',
538
 
            scheme=self._qualified_prefix)
 
550
                                 scheme=self._url_protocol)
539
551
        self.start_server(server)
540
552
        url = server.get_url()
541
 
        http_transport = self._transport(url)
 
553
        # FIXME: needs a cleanup -- vila 20100611
 
554
        http_transport = transport.get_transport_from_url(url)
542
555
        code, response = http_transport._post('abc def end-of-body')
543
556
        self.assertTrue(
544
557
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
545
558
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
 
559
        self.assertTrue('content-type: application/octet-stream\r'
 
560
                        in server.received_bytes.lower())
546
561
        # The transport should not be assuming that the server can accept
547
562
        # chunked encoding the first time it connects, because HTTP/1.1, so we
548
563
        # check for the literal string.
584
599
    Daughter classes are expected to override _req_handler_class
585
600
    """
586
601
 
 
602
    scenarios = multiply_scenarios(
 
603
        vary_by_http_client_implementation(),
 
604
        vary_by_http_protocol_version(),
 
605
        )
 
606
 
587
607
    # Provide a useful default
588
608
    _req_handler_class = http_server.TestingHTTPRequestHandler
589
609
 
590
610
    def create_transport_readonly_server(self):
591
 
        return http_server.HttpServer(self._req_handler_class,
592
 
                                      protocol_version=self._protocol_version)
 
611
        server = http_server.HttpServer(self._req_handler_class,
 
612
                                        protocol_version=self._protocol_version)
 
613
        server._url_protocol = self._url_protocol
 
614
        return server
593
615
 
594
616
    def _testing_pycurl(self):
595
617
        # TODO: This is duplicated for lots of the classes in this file
600
622
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
601
623
    """Whatever request comes in, close the connection"""
602
624
 
603
 
    def handle_one_request(self):
 
625
    def _handle_one_request(self):
604
626
        """Handle a single HTTP request, by abruptly closing the connection"""
605
627
        self.close_connection = 1
606
628
 
611
633
    _req_handler_class = WallRequestHandler
612
634
 
613
635
    def test_http_has(self):
614
 
        server = self.get_readonly_server()
615
 
        t = self._transport(server.get_url())
 
636
        t = self.get_readonly_transport()
616
637
        # Unfortunately httplib (see HTTPResponse._read_status
617
638
        # for details) make no distinction between a closed
618
639
        # socket and badly formatted status line, so we can't
624
645
                          t.has, 'foo/bar')
625
646
 
626
647
    def test_http_get(self):
627
 
        server = self.get_readonly_server()
628
 
        t = self._transport(server.get_url())
 
648
        t = self.get_readonly_transport()
629
649
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
630
650
                           errors.InvalidHttpResponse),
631
651
                          t.get, 'foo/bar')
647
667
 
648
668
    _req_handler_class = BadStatusRequestHandler
649
669
 
 
670
    def setUp(self):
 
671
        super(TestBadStatusServer, self).setUp()
 
672
        # See https://bugs.launchpad.net/bzr/+bug/1451448 for details.
 
673
        # TD;LR: Running both a TCP client and server in the same process and
 
674
        # thread uncovers a race in python. The fix is to run the server in a
 
675
        # different process. Trying to fix yet another race here is not worth
 
676
        # the effort. -- vila 2015-09-06
 
677
        if 'HTTP/1.0' in self.id():
 
678
            raise tests.TestSkipped(
 
679
                'Client/Server in the same process and thread can hang')
 
680
 
650
681
    def test_http_has(self):
651
 
        server = self.get_readonly_server()
652
 
        t = self._transport(server.get_url())
653
 
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
 
682
        t = self.get_readonly_transport()
 
683
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
684
                           errors.InvalidHttpResponse),
 
685
                          t.has, 'foo/bar')
654
686
 
655
687
    def test_http_get(self):
656
 
        server = self.get_readonly_server()
657
 
        t = self._transport(server.get_url())
658
 
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
 
688
        t = self.get_readonly_transport()
 
689
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
690
                           errors.InvalidHttpResponse),
 
691
                          t.get, 'foo/bar')
659
692
 
660
693
 
661
694
class InvalidStatusRequestHandler(http_server.TestingHTTPRequestHandler):
665
698
        """Fakes handling a single HTTP request, returns a bad status"""
666
699
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
667
700
        self.wfile.write("Invalid status line\r\n")
 
701
        # If we don't close the connection pycurl will hang. Since this is a
 
702
        # stress test we don't *have* to respect the protocol, but we don't
 
703
        # have to sabotage it too much either.
 
704
        self.close_connection = True
668
705
        return False
669
706
 
670
707
 
676
713
 
677
714
    _req_handler_class = InvalidStatusRequestHandler
678
715
 
679
 
    def test_http_has(self):
680
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
681
 
            raise tests.KnownFailure(
682
 
                'pycurl hangs if the server send back garbage')
683
 
        super(TestInvalidStatusServer, self).test_http_has()
684
 
 
685
 
    def test_http_get(self):
686
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
687
 
            raise tests.KnownFailure(
688
 
                'pycurl hangs if the server send back garbage')
689
 
        super(TestInvalidStatusServer, self).test_http_get()
690
 
 
691
716
 
692
717
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
693
718
    """Whatever request comes in, returns a bad protocol version"""
715
740
        super(TestBadProtocolServer, self).setUp()
716
741
 
717
742
    def test_http_has(self):
718
 
        server = self.get_readonly_server()
719
 
        t = self._transport(server.get_url())
 
743
        t = self.get_readonly_transport()
720
744
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
721
745
 
722
746
    def test_http_get(self):
723
 
        server = self.get_readonly_server()
724
 
        t = self._transport(server.get_url())
 
747
        t = self.get_readonly_transport()
725
748
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
726
749
 
727
750
 
741
764
    _req_handler_class = ForbiddenRequestHandler
742
765
 
743
766
    def test_http_has(self):
744
 
        server = self.get_readonly_server()
745
 
        t = self._transport(server.get_url())
 
767
        t = self.get_readonly_transport()
746
768
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
747
769
 
748
770
    def test_http_get(self):
749
 
        server = self.get_readonly_server()
750
 
        t = self._transport(server.get_url())
 
771
        t = self.get_readonly_transport()
751
772
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
752
773
 
753
774
 
792
813
        self.build_tree_contents([('a', '0123456789')],)
793
814
 
794
815
    def test_readv(self):
795
 
        server = self.get_readonly_server()
796
 
        t = self._transport(server.get_url())
 
816
        t = self.get_readonly_transport()
797
817
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
798
818
        self.assertEqual(l[0], (0, '0'))
799
819
        self.assertEqual(l[1], (1, '1'))
801
821
        self.assertEqual(l[3], (9, '9'))
802
822
 
803
823
    def test_readv_out_of_order(self):
804
 
        server = self.get_readonly_server()
805
 
        t = self._transport(server.get_url())
 
824
        t = self.get_readonly_transport()
806
825
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
807
826
        self.assertEqual(l[0], (1, '1'))
808
827
        self.assertEqual(l[1], (9, '9'))
810
829
        self.assertEqual(l[3], (3, '34'))
811
830
 
812
831
    def test_readv_invalid_ranges(self):
813
 
        server = self.get_readonly_server()
814
 
        t = self._transport(server.get_url())
 
832
        t = self.get_readonly_transport()
815
833
 
816
834
        # This is intentionally reading off the end of the file
817
835
        # since we are sure that it cannot get there
825
843
 
826
844
    def test_readv_multiple_get_requests(self):
827
845
        server = self.get_readonly_server()
828
 
        t = self._transport(server.get_url())
 
846
        t = self.get_readonly_transport()
829
847
        # force transport to issue multiple requests
830
848
        t._max_readv_combine = 1
831
849
        t._max_get_ranges = 1
839
857
 
840
858
    def test_readv_get_max_size(self):
841
859
        server = self.get_readonly_server()
842
 
        t = self._transport(server.get_url())
 
860
        t = self.get_readonly_transport()
843
861
        # force transport to issue multiple requests by limiting the number of
844
862
        # bytes by request. Note that this apply to coalesced offsets only, a
845
863
        # single range will keep its size even if bigger than the limit.
854
872
 
855
873
    def test_complete_readv_leave_pipe_clean(self):
856
874
        server = self.get_readonly_server()
857
 
        t = self._transport(server.get_url())
 
875
        t = self.get_readonly_transport()
858
876
        # force transport to issue multiple requests
859
877
        t._get_max_size = 2
860
 
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
 
878
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
861
879
        # The server should have issued 3 requests
862
880
        self.assertEqual(3, server.GET_request_nb)
863
881
        self.assertEqual('0123456789', t.get_bytes('a'))
865
883
 
866
884
    def test_incomplete_readv_leave_pipe_clean(self):
867
885
        server = self.get_readonly_server()
868
 
        t = self._transport(server.get_url())
 
886
        t = self.get_readonly_transport()
869
887
        # force transport to issue multiple requests
870
888
        t._get_max_size = 2
871
889
        # Don't collapse readv results into a list so that we leave unread
940
958
    def get_multiple_ranges(self, file, file_size, ranges):
941
959
        self.send_response(206)
942
960
        self.send_header('Accept-Ranges', 'bytes')
 
961
        # XXX: this is strange; the 'random' name below seems undefined and
 
962
        # yet the tests pass -- mbp 2010-10-11 bug 658773
943
963
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
944
964
        self.send_header("Content-Type",
945
965
                         "multipart/byteranges; boundary=%s" % boundary)
1007
1027
                return
1008
1028
            self.send_range_content(file, start, end - start + 1)
1009
1029
            cur += 1
1010
 
        # No final boundary
 
1030
        # Final boundary
1011
1031
        self.wfile.write(boundary_line)
1012
1032
 
1013
1033
 
1021
1041
 
1022
1042
    def test_readv_with_short_reads(self):
1023
1043
        server = self.get_readonly_server()
1024
 
        t = self._transport(server.get_url())
 
1044
        t = self.get_readonly_transport()
1025
1045
        # Force separate ranges for each offset
1026
1046
        t._bytes_to_read_before_seek = 0
1027
1047
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1042
1062
        # that mode
1043
1063
        self.assertEqual('single', t._range_hint)
1044
1064
 
 
1065
 
 
1066
class TruncatedBeforeBoundaryRequestHandler(
 
1067
    http_server.TestingHTTPRequestHandler):
 
1068
    """Truncation before a boundary, like in bug 198646"""
 
1069
 
 
1070
    _truncated_ranges = 1
 
1071
 
 
1072
    def get_multiple_ranges(self, file, file_size, ranges):
 
1073
        self.send_response(206)
 
1074
        self.send_header('Accept-Ranges', 'bytes')
 
1075
        boundary = 'tagada'
 
1076
        self.send_header('Content-Type',
 
1077
                         'multipart/byteranges; boundary=%s' % boundary)
 
1078
        boundary_line = '--%s\r\n' % boundary
 
1079
        # Calculate the Content-Length
 
1080
        content_length = 0
 
1081
        for (start, end) in ranges:
 
1082
            content_length += len(boundary_line)
 
1083
            content_length += self._header_line_length(
 
1084
                'Content-type', 'application/octet-stream')
 
1085
            content_length += self._header_line_length(
 
1086
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
 
1087
            content_length += len('\r\n') # end headers
 
1088
            content_length += end - start # + 1
 
1089
        content_length += len(boundary_line)
 
1090
        self.send_header('Content-length', content_length)
 
1091
        self.end_headers()
 
1092
 
 
1093
        # Send the multipart body
 
1094
        cur = 0
 
1095
        for (start, end) in ranges:
 
1096
            if cur + self._truncated_ranges >= len(ranges):
 
1097
                # Abruptly ends the response and close the connection
 
1098
                self.close_connection = 1
 
1099
                return
 
1100
            self.wfile.write(boundary_line)
 
1101
            self.send_header('Content-type', 'application/octet-stream')
 
1102
            self.send_header('Content-Range', 'bytes %d-%d/%d'
 
1103
                             % (start, end, file_size))
 
1104
            self.end_headers()
 
1105
            self.send_range_content(file, start, end - start + 1)
 
1106
            cur += 1
 
1107
        # Final boundary
 
1108
        self.wfile.write(boundary_line)
 
1109
 
 
1110
 
 
1111
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
 
1112
    """Tests the case of bug 198646, disconnecting before a boundary."""
 
1113
 
 
1114
    _req_handler_class = TruncatedBeforeBoundaryRequestHandler
 
1115
 
 
1116
    def setUp(self):
 
1117
        super(TestTruncatedBeforeBoundary, self).setUp()
 
1118
        self.build_tree_contents([('a', '0123456789')],)
 
1119
 
 
1120
    def test_readv_with_short_reads(self):
 
1121
        server = self.get_readonly_server()
 
1122
        t = self.get_readonly_transport()
 
1123
        # Force separate ranges for each offset
 
1124
        t._bytes_to_read_before_seek = 0
 
1125
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
 
1126
        self.assertEqual((0, '0'), ireadv.next())
 
1127
        self.assertEqual((2, '2'), ireadv.next())
 
1128
        self.assertEqual((4, '45'), ireadv.next())
 
1129
        self.assertEqual((9, '9'), ireadv.next())
 
1130
 
 
1131
 
1045
1132
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1046
1133
    """Errors out when range specifiers exceed the limit"""
1047
1134
 
1071
1158
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1072
1159
    """Tests readv requests against a server erroring out on too much ranges."""
1073
1160
 
 
1161
    scenarios = multiply_scenarios(
 
1162
        vary_by_http_client_implementation(),
 
1163
        vary_by_http_protocol_version(),
 
1164
        )
 
1165
 
1074
1166
    # Requests with more range specifiers will error out
1075
1167
    range_limit = 3
1076
1168
 
1078
1170
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1079
1171
                                      protocol_version=self._protocol_version)
1080
1172
 
1081
 
    def get_transport(self):
1082
 
        return self._transport(self.get_readonly_server().get_url())
1083
 
 
1084
1173
    def setUp(self):
1085
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1174
        super(TestLimitedRangeRequestServer, self).setUp()
1086
1175
        # We need to manipulate ranges that correspond to real chunks in the
1087
1176
        # response, so we build a content appropriately.
1088
1177
        filler = ''.join(['abcdefghij' for x in range(102)])
1090
1179
        self.build_tree_contents([('a', content)],)
1091
1180
 
1092
1181
    def test_few_ranges(self):
1093
 
        t = self.get_transport()
 
1182
        t = self.get_readonly_transport()
1094
1183
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1095
1184
        self.assertEqual(l[0], (0, '0000'))
1096
1185
        self.assertEqual(l[1], (1024, '0001'))
1097
1186
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1098
1187
 
1099
1188
    def test_more_ranges(self):
1100
 
        t = self.get_transport()
 
1189
        t = self.get_readonly_transport()
1101
1190
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1102
1191
        self.assertEqual(l[0], (0, '0000'))
1103
1192
        self.assertEqual(l[1], (1024, '0001'))
1114
1203
    Only the urllib implementation is tested here.
1115
1204
    """
1116
1205
 
1117
 
    def setUp(self):
1118
 
        tests.TestCase.setUp(self)
1119
 
        self._old_env = {}
1120
 
 
1121
 
    def tearDown(self):
1122
 
        self._restore_env()
1123
 
        tests.TestCase.tearDown(self)
1124
 
 
1125
 
    def _install_env(self, env):
1126
 
        for name, value in env.iteritems():
1127
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1128
 
 
1129
 
    def _restore_env(self):
1130
 
        for name, value in self._old_env.iteritems():
1131
 
            osutils.set_or_unset_env(name, value)
1132
 
 
1133
1206
    def _proxied_request(self):
1134
1207
        handler = _urllib2_wrappers.ProxyHandler()
1135
 
        request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
 
1208
        request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
1136
1209
        handler.set_proxy(request, 'http')
1137
1210
        return request
1138
1211
 
 
1212
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
 
1213
        handler = _urllib2_wrappers.ProxyHandler()
 
1214
        self.assertEqual(expected,
 
1215
                          handler.evaluate_proxy_bypass(host, no_proxy))
 
1216
 
1139
1217
    def test_empty_user(self):
1140
 
        self._install_env({'http_proxy': 'http://bar.com'})
1141
 
        request = self._proxied_request()
1142
 
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
 
1218
        self.overrideEnv('http_proxy', 'http://bar.com')
 
1219
        request = self._proxied_request()
 
1220
        self.assertFalse('Proxy-authorization' in request.headers)
 
1221
 
 
1222
    def test_user_with_at(self):
 
1223
        self.overrideEnv('http_proxy',
 
1224
                         'http://username@domain:password@proxy_host:1234')
 
1225
        request = self._proxied_request()
 
1226
        self.assertFalse('Proxy-authorization' in request.headers)
1143
1227
 
1144
1228
    def test_invalid_proxy(self):
1145
1229
        """A proxy env variable without scheme"""
1146
 
        self._install_env({'http_proxy': 'host:1234'})
 
1230
        self.overrideEnv('http_proxy', 'host:1234')
1147
1231
        self.assertRaises(errors.InvalidURL, self._proxied_request)
1148
1232
 
 
1233
    def test_evaluate_proxy_bypass_true(self):
 
1234
        """The host is not proxied"""
 
1235
        self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
 
1236
        self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
 
1237
 
 
1238
    def test_evaluate_proxy_bypass_false(self):
 
1239
        """The host is proxied"""
 
1240
        self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
 
1241
 
 
1242
    def test_evaluate_proxy_bypass_unknown(self):
 
1243
        """The host is not explicitly proxied"""
 
1244
        self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
 
1245
        self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
 
1246
 
 
1247
    def test_evaluate_proxy_bypass_empty_entries(self):
 
1248
        """Ignore empty entries"""
 
1249
        self.assertEvaluateProxyBypass(None, 'example.com', '')
 
1250
        self.assertEvaluateProxyBypass(None, 'example.com', ',')
 
1251
        self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
 
1252
 
1149
1253
 
1150
1254
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
1151
1255
    """Tests proxy server.
1156
1260
    to the file names).
1157
1261
    """
1158
1262
 
 
1263
    scenarios = multiply_scenarios(
 
1264
        vary_by_http_client_implementation(),
 
1265
        vary_by_http_protocol_version(),
 
1266
        )
 
1267
 
1159
1268
    # FIXME: We don't have an https server available, so we don't
1160
 
    # test https connections.
 
1269
    # test https connections. --vila toolongago
1161
1270
 
1162
1271
    def setUp(self):
1163
1272
        super(TestProxyHttpServer, self).setUp()
 
1273
        self.transport_secondary_server = http_utils.ProxyServer
1164
1274
        self.build_tree_contents([('foo', 'contents of foo\n'),
1165
1275
                                  ('foo-proxied', 'proxied contents of foo\n')])
1166
1276
        # Let's setup some attributes for tests
1167
 
        self.server = self.get_readonly_server()
1168
 
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
 
1277
        server = self.get_readonly_server()
 
1278
        self.server_host_port = '%s:%d' % (server.host, server.port)
1169
1279
        if self._testing_pycurl():
1170
1280
            # Oh my ! pycurl does not check for the port as part of
1171
1281
            # no_proxy :-( So we just test the host part
1172
 
            self.no_proxy_host = self.server.host
 
1282
            self.no_proxy_host = server.host
1173
1283
        else:
1174
 
            self.no_proxy_host = self.proxy_address
 
1284
            self.no_proxy_host = self.server_host_port
1175
1285
        # The secondary server is the proxy
1176
 
        self.proxy = self.get_secondary_server()
1177
 
        self.proxy_url = self.proxy.get_url()
1178
 
        self._old_env = {}
 
1286
        self.proxy_url = self.get_secondary_url()
 
1287
        if self._testing_pycurl():
 
1288
            self.proxy_url = self.proxy_url.replace('+pycurl', '')
1179
1289
 
1180
1290
    def _testing_pycurl(self):
1181
1291
        # TODO: This is duplicated for lots of the classes in this file
1182
1292
        return (features.pycurl.available()
1183
1293
                and self._transport == PyCurlTransport)
1184
1294
 
1185
 
    def create_transport_secondary_server(self):
1186
 
        """Creates an http server that will serve files with
1187
 
        '-proxied' appended to their names.
1188
 
        """
1189
 
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
1190
 
 
1191
 
    def _install_env(self, env):
1192
 
        for name, value in env.iteritems():
1193
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1194
 
 
1195
 
    def _restore_env(self):
1196
 
        for name, value in self._old_env.iteritems():
1197
 
            osutils.set_or_unset_env(name, value)
1198
 
 
1199
 
    def proxied_in_env(self, env):
1200
 
        self._install_env(env)
1201
 
        url = self.server.get_url()
1202
 
        t = self._transport(url)
1203
 
        try:
1204
 
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1205
 
        finally:
1206
 
            self._restore_env()
1207
 
 
1208
 
    def not_proxied_in_env(self, env):
1209
 
        self._install_env(env)
1210
 
        url = self.server.get_url()
1211
 
        t = self._transport(url)
1212
 
        try:
1213
 
            self.assertEqual('contents of foo\n', t.get('foo').read())
1214
 
        finally:
1215
 
            self._restore_env()
 
1295
    def assertProxied(self):
 
1296
        t = self.get_readonly_transport()
 
1297
        self.assertEqual('proxied contents of foo\n', t.get('foo').read())
 
1298
 
 
1299
    def assertNotProxied(self):
 
1300
        t = self.get_readonly_transport()
 
1301
        self.assertEqual('contents of foo\n', t.get('foo').read())
1216
1302
 
1217
1303
    def test_http_proxy(self):
1218
 
        self.proxied_in_env({'http_proxy': self.proxy_url})
 
1304
        self.overrideEnv('http_proxy', self.proxy_url)
 
1305
        self.assertProxied()
1219
1306
 
1220
1307
    def test_HTTP_PROXY(self):
1221
1308
        if self._testing_pycurl():
1224
1311
            # about. Should we ?)
1225
1312
            raise tests.TestNotApplicable(
1226
1313
                'pycurl does not check HTTP_PROXY for security reasons')
1227
 
        self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
 
1314
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1315
        self.assertProxied()
1228
1316
 
1229
1317
    def test_all_proxy(self):
1230
 
        self.proxied_in_env({'all_proxy': self.proxy_url})
 
1318
        self.overrideEnv('all_proxy', self.proxy_url)
 
1319
        self.assertProxied()
1231
1320
 
1232
1321
    def test_ALL_PROXY(self):
1233
 
        self.proxied_in_env({'ALL_PROXY': self.proxy_url})
 
1322
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1323
        self.assertProxied()
1234
1324
 
1235
1325
    def test_http_proxy_with_no_proxy(self):
1236
 
        self.not_proxied_in_env({'http_proxy': self.proxy_url,
1237
 
                                 'no_proxy': self.no_proxy_host})
 
1326
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1327
        self.overrideEnv('http_proxy', self.proxy_url)
 
1328
        self.assertNotProxied()
1238
1329
 
1239
1330
    def test_HTTP_PROXY_with_NO_PROXY(self):
1240
1331
        if self._testing_pycurl():
1241
1332
            raise tests.TestNotApplicable(
1242
1333
                'pycurl does not check HTTP_PROXY for security reasons')
1243
 
        self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
1244
 
                                 'NO_PROXY': self.no_proxy_host})
 
1334
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1335
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1336
        self.assertNotProxied()
1245
1337
 
1246
1338
    def test_all_proxy_with_no_proxy(self):
1247
 
        self.not_proxied_in_env({'all_proxy': self.proxy_url,
1248
 
                                 'no_proxy': self.no_proxy_host})
 
1339
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1340
        self.overrideEnv('all_proxy', self.proxy_url)
 
1341
        self.assertNotProxied()
1249
1342
 
1250
1343
    def test_ALL_PROXY_with_NO_PROXY(self):
1251
 
        self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
1252
 
                                 'NO_PROXY': self.no_proxy_host})
 
1344
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1345
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1346
        self.assertNotProxied()
1253
1347
 
1254
1348
    def test_http_proxy_without_scheme(self):
 
1349
        self.overrideEnv('http_proxy', self.server_host_port)
1255
1350
        if self._testing_pycurl():
1256
1351
            # pycurl *ignores* invalid proxy env variables. If that ever change
1257
1352
            # in the future, this test will fail indicating that pycurl do not
1258
1353
            # ignore anymore such variables.
1259
 
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
 
1354
            self.assertNotProxied()
1260
1355
        else:
1261
 
            self.assertRaises(errors.InvalidURL,
1262
 
                              self.proxied_in_env,
1263
 
                              {'http_proxy': self.proxy_address})
 
1356
            self.assertRaises(errors.InvalidURL, self.assertProxied)
1264
1357
 
1265
1358
 
1266
1359
class TestRanges(http_utils.TestCaseWithWebserver):
1267
1360
    """Test the Range header in GET methods."""
1268
1361
 
 
1362
    scenarios = multiply_scenarios(
 
1363
        vary_by_http_client_implementation(),
 
1364
        vary_by_http_protocol_version(),
 
1365
        )
 
1366
 
1269
1367
    def setUp(self):
1270
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1368
        super(TestRanges, self).setUp()
1271
1369
        self.build_tree_contents([('a', '0123456789')],)
1272
 
        server = self.get_readonly_server()
1273
 
        self.transport = self._transport(server.get_url())
1274
1370
 
1275
1371
    def create_transport_readonly_server(self):
1276
1372
        return http_server.HttpServer(protocol_version=self._protocol_version)
1277
1373
 
1278
1374
    def _file_contents(self, relpath, ranges):
 
1375
        t = self.get_readonly_transport()
1279
1376
        offsets = [ (start, end - start + 1) for start, end in ranges]
1280
 
        coalesce = self.transport._coalesce_offsets
 
1377
        coalesce = t._coalesce_offsets
1281
1378
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1282
 
        code, data = self.transport._get(relpath, coalesced)
 
1379
        code, data = t._get(relpath, coalesced)
1283
1380
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1284
1381
        for start, end in ranges:
1285
1382
            data.seek(start)
1286
1383
            yield data.read(end - start + 1)
1287
1384
 
1288
1385
    def _file_tail(self, relpath, tail_amount):
1289
 
        code, data = self.transport._get(relpath, [], tail_amount)
 
1386
        t = self.get_readonly_transport()
 
1387
        code, data = t._get(relpath, [], tail_amount)
1290
1388
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1291
1389
        data.seek(-tail_amount, 2)
1292
1390
        return data.read(tail_amount)
1311
1409
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1312
1410
    """Test redirection between http servers."""
1313
1411
 
1314
 
    def create_transport_secondary_server(self):
1315
 
        """Create the secondary server redirecting to the primary server"""
1316
 
        new = self.get_readonly_server()
1317
 
 
1318
 
        redirecting = http_utils.HTTPServerRedirecting(
1319
 
            protocol_version=self._protocol_version)
1320
 
        redirecting.redirect_to(new.host, new.port)
1321
 
        return redirecting
 
1412
    scenarios = multiply_scenarios(
 
1413
        vary_by_http_client_implementation(),
 
1414
        vary_by_http_protocol_version(),
 
1415
        )
1322
1416
 
1323
1417
    def setUp(self):
1324
1418
        super(TestHTTPRedirections, self).setUp()
1326
1420
                                  ('bundle',
1327
1421
                                  '# Bazaar revision bundle v0.9\n#\n')
1328
1422
                                  ],)
1329
 
        # The requests to the old server will be redirected to the new server
1330
 
        self.old_transport = self._transport(self.old_server.get_url())
1331
1423
 
1332
1424
    def test_redirected(self):
1333
 
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1334
 
        t = self._transport(self.new_server.get_url())
1335
 
        self.assertEqual('0123456789', t.get('a').read())
1336
 
 
1337
 
    def test_read_redirected_bundle_from_url(self):
1338
 
        from bzrlib.bundle import read_bundle_from_url
1339
 
        url = self.old_transport.abspath('bundle')
1340
 
        bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
1341
 
                read_bundle_from_url, url)
1342
 
        # If read_bundle_from_url was successful we get an empty bundle
1343
 
        self.assertEqual([], bundle.revisions)
 
1425
        self.assertRaises(errors.RedirectRequested,
 
1426
                          self.get_old_transport().get, 'a')
 
1427
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1344
1428
 
1345
1429
 
1346
1430
class RedirectedRequest(_urllib2_wrappers.Request):
1363
1447
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1364
1448
 
1365
1449
 
 
1450
def cleanup_http_redirection_connections(test):
 
1451
    # Some sockets are opened but never seen by _urllib, so we trap them at
 
1452
    # the _urllib2_wrappers level to be able to clean them up.
 
1453
    def socket_disconnect(sock):
 
1454
        try:
 
1455
            sock.shutdown(socket.SHUT_RDWR)
 
1456
            sock.close()
 
1457
        except socket.error:
 
1458
            pass
 
1459
    def connect(connection):
 
1460
        test.http_connect_orig(connection)
 
1461
        test.addCleanup(socket_disconnect, connection.sock)
 
1462
    test.http_connect_orig = test.overrideAttr(
 
1463
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
 
1464
    def connect(connection):
 
1465
        test.https_connect_orig(connection)
 
1466
        test.addCleanup(socket_disconnect, connection.sock)
 
1467
    test.https_connect_orig = test.overrideAttr(
 
1468
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
 
1469
 
 
1470
 
1366
1471
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1367
1472
    """Test redirections.
1368
1473
 
1377
1482
    -- vila 20070212
1378
1483
    """
1379
1484
 
 
1485
    scenarios = multiply_scenarios(
 
1486
        vary_by_http_client_implementation(),
 
1487
        vary_by_http_protocol_version(),
 
1488
        )
 
1489
 
1380
1490
    def setUp(self):
1381
1491
        if (features.pycurl.available()
1382
1492
            and self._transport == PyCurlTransport):
1383
1493
            raise tests.TestNotApplicable(
1384
 
                "pycurl doesn't redirect silently annymore")
 
1494
                "pycurl doesn't redirect silently anymore")
1385
1495
        super(TestHTTPSilentRedirections, self).setUp()
1386
1496
        install_redirected_request(self)
 
1497
        cleanup_http_redirection_connections(self)
1387
1498
        self.build_tree_contents([('a','a'),
1388
1499
                                  ('1/',),
1389
1500
                                  ('1/a', 'redirected once'),
1397
1508
                                  ('5/a', 'redirected 5 times'),
1398
1509
                                  ],)
1399
1510
 
1400
 
        self.old_transport = self._transport(self.old_server.get_url())
1401
 
 
1402
 
    def create_transport_secondary_server(self):
1403
 
        """Create the secondary server, redirections are defined in the tests"""
1404
 
        return http_utils.HTTPServerRedirecting(
1405
 
            protocol_version=self._protocol_version)
1406
 
 
1407
1511
    def test_one_redirection(self):
1408
 
        t = self.old_transport
1409
 
 
1410
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1512
        t = self.get_old_transport()
 
1513
        req = RedirectedRequest('GET', t._remote_path('a'))
1411
1514
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1412
1515
                                       self.new_server.port)
1413
1516
        self.old_server.redirections = \
1414
1517
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1415
 
        self.assertEqual('redirected once',t._perform(req).read())
 
1518
        self.assertEqual('redirected once', t._perform(req).read())
1416
1519
 
1417
1520
    def test_five_redirections(self):
1418
 
        t = self.old_transport
1419
 
 
1420
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1521
        t = self.get_old_transport()
 
1522
        req = RedirectedRequest('GET', t._remote_path('a'))
1421
1523
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1422
1524
                                       self.old_server.port)
1423
1525
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1429
1531
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1430
1532
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1431
1533
            ]
1432
 
        self.assertEqual('redirected 5 times',t._perform(req).read())
 
1534
        self.assertEqual('redirected 5 times', t._perform(req).read())
1433
1535
 
1434
1536
 
1435
1537
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1436
1538
    """Test transport.do_catching_redirections."""
1437
1539
 
 
1540
    scenarios = multiply_scenarios(
 
1541
        vary_by_http_client_implementation(),
 
1542
        vary_by_http_protocol_version(),
 
1543
        )
 
1544
 
1438
1545
    def setUp(self):
1439
1546
        super(TestDoCatchRedirections, self).setUp()
1440
1547
        self.build_tree_contents([('a', '0123456789'),],)
1441
 
 
1442
 
        self.old_transport = self._transport(self.old_server.get_url())
1443
 
 
1444
 
    def get_a(self, transport):
1445
 
        return transport.get('a')
 
1548
        cleanup_http_redirection_connections(self)
 
1549
 
 
1550
        self.old_transport = self.get_old_transport()
 
1551
 
 
1552
    def get_a(self, t):
 
1553
        return t.get('a')
1446
1554
 
1447
1555
    def test_no_redirection(self):
1448
 
        t = self._transport(self.new_server.get_url())
 
1556
        t = self.get_new_transport()
1449
1557
 
1450
1558
        # We use None for redirected so that we fail if redirected
1451
1559
        self.assertEqual('0123456789',
1455
1563
    def test_one_redirection(self):
1456
1564
        self.redirections = 0
1457
1565
 
1458
 
        def redirected(transport, exception, redirection_notice):
 
1566
        def redirected(t, exception, redirection_notice):
1459
1567
            self.redirections += 1
1460
 
            dir, file = urlutils.split(exception.target)
1461
 
            return self._transport(dir)
 
1568
            redirected_t = t._redirected_to(exception.source, exception.target)
 
1569
            return redirected_t
1462
1570
 
1463
1571
        self.assertEqual('0123456789',
1464
1572
                         transport.do_catching_redirections(
1478
1586
                          self.get_a, self.old_transport, redirected)
1479
1587
 
1480
1588
 
 
1589
def _setup_authentication_config(**kwargs):
 
1590
    conf = config.AuthenticationConfig()
 
1591
    conf._get_config().update({'httptest': kwargs})
 
1592
    conf._save()
 
1593
 
 
1594
 
 
1595
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
 
1596
    """Unit tests for glue by which urllib2 asks us for authentication"""
 
1597
 
 
1598
    def test_get_user_password_without_port(self):
 
1599
        """We cope if urllib2 doesn't tell us the port.
 
1600
 
 
1601
        See https://bugs.launchpad.net/bzr/+bug/654684
 
1602
        """
 
1603
        user = 'joe'
 
1604
        password = 'foo'
 
1605
        _setup_authentication_config(scheme='http', host='localhost',
 
1606
                                     user=user, password=password)
 
1607
        handler = _urllib2_wrappers.HTTPAuthHandler()
 
1608
        got_pass = handler.get_user_password(dict(
 
1609
            user='joe',
 
1610
            protocol='http',
 
1611
            host='localhost',
 
1612
            path='/',
 
1613
            realm='Realm',
 
1614
            ))
 
1615
        self.assertEqual((user, password), got_pass)
 
1616
 
 
1617
 
1481
1618
class TestAuth(http_utils.TestCaseWithWebserver):
1482
1619
    """Test authentication scheme"""
1483
1620
 
1484
 
    _auth_header = 'Authorization'
1485
 
    _password_prompt_prefix = ''
1486
 
    _username_prompt_prefix = ''
1487
 
    # Set by load_tests
1488
 
    _auth_server = None
 
1621
    scenarios = multiply_scenarios(
 
1622
        vary_by_http_client_implementation(),
 
1623
        vary_by_http_protocol_version(),
 
1624
        vary_by_http_auth_scheme(),
 
1625
        )
1489
1626
 
1490
1627
    def setUp(self):
1491
1628
        super(TestAuth, self).setUp()
1494
1631
                                  ('b', 'contents of b\n'),])
1495
1632
 
1496
1633
    def create_transport_readonly_server(self):
1497
 
        return self._auth_server(protocol_version=self._protocol_version)
 
1634
        server = self._auth_server(protocol_version=self._protocol_version)
 
1635
        server._url_protocol = self._url_protocol
 
1636
        return server
1498
1637
 
1499
1638
    def _testing_pycurl(self):
1500
1639
        # TODO: This is duplicated for lots of the classes in this file
1513
1652
        return url
1514
1653
 
1515
1654
    def get_user_transport(self, user, password):
1516
 
        return self._transport(self.get_user_url(user, password))
 
1655
        t = transport.get_transport_from_url(
 
1656
            self.get_user_url(user, password))
 
1657
        return t
1517
1658
 
1518
1659
    def test_no_user(self):
1519
1660
        self.server.add_user('joe', 'foo')
1561
1702
 
1562
1703
        self.server.add_user('joe', 'foo')
1563
1704
        t = self.get_user_transport(None, None)
1564
 
        stdout = tests.StringIOWrapper()
1565
 
        stderr = tests.StringIOWrapper()
1566
 
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
1567
 
                                            stdout=stdout, stderr=stderr)
 
1705
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n')
 
1706
        stdout, stderr = ui.ui_factory.stdout, ui.ui_factory.stderr
1568
1707
        self.assertEqual('contents of a\n',t.get('a').read())
1569
1708
        # stdin should be empty
1570
1709
        self.assertEqual('', ui.ui_factory.stdin.readline())
1583
1722
 
1584
1723
        self.server.add_user('joe', 'foo')
1585
1724
        t = self.get_user_transport('joe', None)
1586
 
        stdout = tests.StringIOWrapper()
1587
 
        stderr = tests.StringIOWrapper()
1588
 
        ui.ui_factory = tests.TestUIFactory(stdin='foo\n',
1589
 
                                            stdout=stdout, stderr=stderr)
 
1725
        ui.ui_factory = tests.TestUIFactory(stdin='foo\n')
 
1726
        stdout, stderr = ui.ui_factory.stdout, ui.ui_factory.stderr
1590
1727
        self.assertEqual('contents of a\n', t.get('a').read())
1591
1728
        # stdin should be empty
1592
1729
        self.assertEqual('', ui.ui_factory.stdin.readline())
1627
1764
        stdin_content = 'bar\n'  # Not the right password
1628
1765
        self.server.add_user(user, password)
1629
1766
        t = self.get_user_transport(user, None)
1630
 
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1631
 
                                            stderr=tests.StringIOWrapper())
 
1767
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content)
1632
1768
        # Create a minimal config file with the right password
1633
 
        conf = config.AuthenticationConfig()
1634
 
        conf._get_config().update(
1635
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1636
 
                          'user': user, 'password': password}})
1637
 
        conf._save()
 
1769
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1770
                                     user=user, password=password)
1638
1771
        # Issue a request to the server to connect
1639
1772
        self.assertEqual('contents of a\n',t.get('a').read())
1640
1773
        # stdin should have  been left untouched
1642
1775
        # Only one 'Authentication Required' error should occur
1643
1776
        self.assertEqual(1, self.server.auth_required_errors)
1644
1777
 
1645
 
    def test_user_from_auth_conf(self):
1646
 
        if self._testing_pycurl():
1647
 
            raise tests.TestNotApplicable(
1648
 
                'pycurl does not support authentication.conf')
1649
 
        user = 'joe'
1650
 
        password = 'foo'
1651
 
        self.server.add_user(user, password)
1652
 
        # Create a minimal config file with the right password
1653
 
        conf = config.AuthenticationConfig()
1654
 
        conf._get_config().update(
1655
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1656
 
                          'user': user, 'password': password}})
1657
 
        conf._save()
1658
 
        t = self.get_user_transport(None, None)
1659
 
        # Issue a request to the server to connect
1660
 
        self.assertEqual('contents of a\n', t.get('a').read())
1661
 
        # Only one 'Authentication Required' error should occur
1662
 
        self.assertEqual(1, self.server.auth_required_errors)
1663
 
 
1664
1778
    def test_changing_nonce(self):
1665
1779
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1666
1780
                                     http_utils.ProxyDigestAuthServer):
1667
1781
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1668
1782
        if self._testing_pycurl():
1669
 
            raise tests.KnownFailure(
 
1783
            self.knownFailure(
1670
1784
                'pycurl does not handle a nonce change')
1671
1785
        self.server.add_user('joe', 'foo')
1672
1786
        t = self.get_user_transport('joe', 'foo')
1682
1796
        # initial 'who are you' and a second 'who are you' with the new nonce)
1683
1797
        self.assertEqual(2, self.server.auth_required_errors)
1684
1798
 
 
1799
    def test_user_from_auth_conf(self):
 
1800
        if self._testing_pycurl():
 
1801
            raise tests.TestNotApplicable(
 
1802
                'pycurl does not support authentication.conf')
 
1803
        user = 'joe'
 
1804
        password = 'foo'
 
1805
        self.server.add_user(user, password)
 
1806
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1807
                                     user=user, password=password)
 
1808
        t = self.get_user_transport(None, None)
 
1809
        # Issue a request to the server to connect
 
1810
        self.assertEqual('contents of a\n', t.get('a').read())
 
1811
        # Only one 'Authentication Required' error should occur
 
1812
        self.assertEqual(1, self.server.auth_required_errors)
 
1813
 
 
1814
    def test_no_credential_leaks_in_log(self):
 
1815
        self.overrideAttr(debug, 'debug_flags', {'http'})
 
1816
        user = 'joe'
 
1817
        password = 'very-sensitive-password'
 
1818
        self.server.add_user(user, password)
 
1819
        t = self.get_user_transport(user, password)
 
1820
        # Capture the debug calls to mutter
 
1821
        self.mutters = []
 
1822
        def mutter(*args):
 
1823
            lines = args[0] % args[1:]
 
1824
            # Some calls output multiple lines, just split them now since we
 
1825
            # care about a single one later.
 
1826
            self.mutters.extend(lines.splitlines())
 
1827
        self.overrideAttr(trace, 'mutter', mutter)
 
1828
        # Issue a request to the server to connect
 
1829
        self.assertEqual(True, t.has('a'))
 
1830
        # Only one 'Authentication Required' error should occur
 
1831
        self.assertEqual(1, self.server.auth_required_errors)
 
1832
        # Since the authentification succeeded, there should be a corresponding
 
1833
        # debug line
 
1834
        sent_auth_headers = [line for line in self.mutters
 
1835
                             if line.startswith('> %s' % (self._auth_header,))]
 
1836
        self.assertLength(1, sent_auth_headers)
 
1837
        self.assertStartsWith(sent_auth_headers[0],
 
1838
                              '> %s: <masked>' % (self._auth_header,))
1685
1839
 
1686
1840
 
1687
1841
class TestProxyAuth(TestAuth):
1688
 
    """Test proxy authentication schemes."""
1689
 
 
1690
 
    _auth_header = 'Proxy-authorization'
1691
 
    _password_prompt_prefix = 'Proxy '
1692
 
    _username_prompt_prefix = 'Proxy '
 
1842
    """Test proxy authentication schemes.
 
1843
 
 
1844
    This inherits from TestAuth to tweak the setUp and filter some failing
 
1845
    tests.
 
1846
    """
 
1847
 
 
1848
    scenarios = multiply_scenarios(
 
1849
        vary_by_http_client_implementation(),
 
1850
        vary_by_http_protocol_version(),
 
1851
        vary_by_http_proxy_auth_scheme(),
 
1852
        )
1693
1853
 
1694
1854
    def setUp(self):
1695
1855
        super(TestProxyAuth, self).setUp()
1696
 
        self._old_env = {}
1697
 
        self.addCleanup(self._restore_env)
1698
1856
        # Override the contents to avoid false positives
1699
1857
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1700
1858
                                  ('b', 'not proxied contents of b\n'),
1703
1861
                                  ])
1704
1862
 
1705
1863
    def get_user_transport(self, user, password):
1706
 
        self._install_env({'all_proxy': self.get_user_url(user, password)})
1707
 
        return self._transport(self.server.get_url())
1708
 
 
1709
 
    def _install_env(self, env):
1710
 
        for name, value in env.iteritems():
1711
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1712
 
 
1713
 
    def _restore_env(self):
1714
 
        for name, value in self._old_env.iteritems():
1715
 
            osutils.set_or_unset_env(name, value)
 
1864
        proxy_url = self.get_user_url(user, password)
 
1865
        if self._testing_pycurl():
 
1866
            proxy_url = proxy_url.replace('+pycurl', '')
 
1867
        self.overrideEnv('all_proxy', proxy_url)
 
1868
        return TestAuth.get_user_transport(self, user, password)
1716
1869
 
1717
1870
    def test_empty_pass(self):
1718
1871
        if self._testing_pycurl():
1719
1872
            import pycurl
1720
1873
            if pycurl.version_info()[1] < '7.16.0':
1721
 
                raise tests.KnownFailure(
 
1874
                self.knownFailure(
1722
1875
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1723
1876
        super(TestProxyAuth, self).test_empty_pass()
1724
1877
 
1725
1878
 
 
1879
class NonClosingBytesIO(io.BytesIO):
 
1880
 
 
1881
    def close(self):
 
1882
        """Ignore and leave file open."""
 
1883
 
 
1884
 
1726
1885
class SampleSocket(object):
1727
1886
    """A socket-like object for use in testing the HTTP request handler."""
1728
1887
 
1731
1890
 
1732
1891
        :param socket_read_content: a byte sequence
1733
1892
        """
1734
 
        # Use plain python StringIO so we can monkey-patch the close method to
1735
 
        # not discard the contents.
1736
 
        from StringIO import StringIO
1737
 
        self.readfile = StringIO(socket_read_content)
1738
 
        self.writefile = StringIO()
1739
 
        self.writefile.close = lambda: None
 
1893
        self.readfile = io.BytesIO(socket_read_content)
 
1894
        self.writefile = NonClosingBytesIO()
 
1895
 
 
1896
    def close(self):
 
1897
        """Ignore and leave files alone."""
1740
1898
 
1741
1899
    def makefile(self, mode='r', bufsize=None):
1742
1900
        if 'r' in mode:
1747
1905
 
1748
1906
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1749
1907
 
 
1908
    scenarios = multiply_scenarios(
 
1909
        vary_by_http_client_implementation(),
 
1910
        vary_by_http_protocol_version(),
 
1911
        )
 
1912
 
1750
1913
    def setUp(self):
1751
1914
        super(SmartHTTPTunnellingTest, self).setUp()
1752
1915
        # We use the VFS layer as part of HTTP tunnelling tests.
1753
 
        self._captureVar('BZR_NO_SMART_VFS', None)
 
1916
        self.overrideEnv('BRZ_NO_SMART_VFS', None)
1754
1917
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
 
1918
        self.http_server = self.get_readonly_server()
1755
1919
 
1756
1920
    def create_transport_readonly_server(self):
1757
 
        return http_utils.HTTPServerWithSmarts(
 
1921
        server = http_utils.HTTPServerWithSmarts(
1758
1922
            protocol_version=self._protocol_version)
 
1923
        server._url_protocol = self._url_protocol
 
1924
        return server
1759
1925
 
1760
 
    def test_open_bzrdir(self):
 
1926
    def test_open_controldir(self):
1761
1927
        branch = self.make_branch('relpath')
1762
 
        http_server = self.get_readonly_server()
1763
 
        url = http_server.get_url() + 'relpath'
1764
 
        bd = bzrdir.BzrDir.open(url)
 
1928
        url = self.http_server.get_url() + 'relpath'
 
1929
        bd = controldir.ControlDir.open(url)
 
1930
        self.addCleanup(bd.transport.disconnect)
1765
1931
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1766
1932
 
1767
1933
    def test_bulk_data(self):
1769
1935
        # The 'readv' command in the smart protocol both sends and receives
1770
1936
        # bulk data, so we use that.
1771
1937
        self.build_tree(['data-file'])
1772
 
        http_server = self.get_readonly_server()
1773
 
        http_transport = self._transport(http_server.get_url())
 
1938
        http_transport = transport.get_transport_from_url(
 
1939
            self.http_server.get_url())
1774
1940
        medium = http_transport.get_smart_medium()
1775
1941
        # Since we provide the medium, the url below will be mostly ignored
1776
1942
        # during the test, as long as the path is '/'.
1784
1950
        post_body = 'hello\n'
1785
1951
        expected_reply_body = 'ok\x012\n'
1786
1952
 
1787
 
        http_server = self.get_readonly_server()
1788
 
        http_transport = self._transport(http_server.get_url())
 
1953
        http_transport = transport.get_transport_from_url(
 
1954
            self.http_server.get_url())
1789
1955
        medium = http_transport.get_smart_medium()
1790
1956
        response = medium.send_http_smart_request(post_body)
1791
1957
        reply_body = response.read()
1792
1958
        self.assertEqual(expected_reply_body, reply_body)
1793
1959
 
1794
1960
    def test_smart_http_server_post_request_handler(self):
1795
 
        httpd = self.get_readonly_server()._get_httpd()
 
1961
        httpd = self.http_server.server
1796
1962
 
1797
1963
        socket = SampleSocket(
1798
1964
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1830
1996
 
1831
1997
    def test_probe_smart_server(self):
1832
1998
        """Test error handling against server refusing smart requests."""
1833
 
        server = self.get_readonly_server()
1834
 
        t = self._transport(server.get_url())
 
1999
        t = self.get_readonly_transport()
1835
2000
        # No need to build a valid smart request here, the server will not even
1836
2001
        # try to interpret it.
1837
2002
        self.assertRaises(errors.SmartProtocolError,
1838
2003
                          t.get_smart_medium().send_http_smart_request,
1839
2004
                          'whatever')
1840
2005
 
 
2006
 
1841
2007
class Test_redirected_to(tests.TestCase):
1842
2008
 
 
2009
    scenarios = vary_by_http_client_implementation()
 
2010
 
1843
2011
    def test_redirected_to_subdir(self):
1844
2012
        t = self._transport('http://www.example.com/foo')
1845
2013
        r = t._redirected_to('http://www.example.com/foo',
1847
2015
        self.assertIsInstance(r, type(t))
1848
2016
        # Both transports share the some connection
1849
2017
        self.assertEqual(t._get_connection(), r._get_connection())
 
2018
        self.assertEqual('http://www.example.com/foo/subdir/', r.base)
1850
2019
 
1851
2020
    def test_redirected_to_self_with_slash(self):
1852
2021
        t = self._transport('http://www.example.com/foo')
1863
2032
        r = t._redirected_to('http://www.example.com/foo',
1864
2033
                             'http://foo.example.com/foo/subdir')
1865
2034
        self.assertIsInstance(r, type(t))
 
2035
        self.assertEqual('http://foo.example.com/foo/subdir/',
 
2036
            r.external_url())
1866
2037
 
1867
2038
    def test_redirected_to_same_host_sibling_protocol(self):
1868
2039
        t = self._transport('http://www.example.com/foo')
1869
2040
        r = t._redirected_to('http://www.example.com/foo',
1870
2041
                             'https://www.example.com/foo')
1871
2042
        self.assertIsInstance(r, type(t))
 
2043
        self.assertEqual('https://www.example.com/foo/',
 
2044
            r.external_url())
1872
2045
 
1873
2046
    def test_redirected_to_same_host_different_protocol(self):
1874
2047
        t = self._transport('http://www.example.com/foo')
1875
2048
        r = t._redirected_to('http://www.example.com/foo',
1876
2049
                             'ftp://www.example.com/foo')
1877
 
        self.assertNotEquals(type(r), type(t))
 
2050
        self.assertNotEqual(type(r), type(t))
 
2051
        self.assertEqual('ftp://www.example.com/foo/', r.external_url())
 
2052
 
 
2053
    def test_redirected_to_same_host_specific_implementation(self):
 
2054
        t = self._transport('http://www.example.com/foo')
 
2055
        r = t._redirected_to('http://www.example.com/foo',
 
2056
                             'https+urllib://www.example.com/foo')
 
2057
        self.assertEqual('https://www.example.com/foo/', r.external_url())
1878
2058
 
1879
2059
    def test_redirected_to_different_host_same_user(self):
1880
2060
        t = self._transport('http://joe@www.example.com/foo')
1881
2061
        r = t._redirected_to('http://www.example.com/foo',
1882
2062
                             'https://foo.example.com/foo')
1883
2063
        self.assertIsInstance(r, type(t))
1884
 
        self.assertEqual(t._user, r._user)
 
2064
        self.assertEqual(t._parsed_url.user, r._parsed_url.user)
 
2065
        self.assertEqual('https://joe@foo.example.com/foo/', r.external_url())
1885
2066
 
1886
2067
 
1887
2068
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1896
2077
    line.
1897
2078
    """
1898
2079
 
1899
 
    def handle_one_request(self):
 
2080
    def _handle_one_request(self):
1900
2081
        tcs = self.server.test_case_server
1901
2082
        requestline = self.rfile.readline()
1902
2083
        headers = self.MessageClass(self.rfile, 0)
1940
2121
    pass
1941
2122
 
1942
2123
 
1943
 
if tests.HTTPSServerFeature.available():
1944
 
    from bzrlib.tests import https_server
 
2124
if features.HTTPSServerFeature.available():
 
2125
    from . import https_server
1945
2126
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
1946
2127
        pass
1947
2128
 
1954
2135
    """
1955
2136
 
1956
2137
    def setUp(self):
1957
 
        tests.TestCase.setUp(self)
1958
2138
        self.server = self._activity_server(self._protocol_version)
1959
2139
        self.server.start_server()
1960
 
        self.activities = {}
 
2140
        self.addCleanup(self.server.stop_server)
 
2141
        _activities = {} # Don't close over self and create a cycle
1961
2142
        def report_activity(t, bytes, direction):
1962
 
            count = self.activities.get(direction, 0)
 
2143
            count = _activities.get(direction, 0)
1963
2144
            count += bytes
1964
 
            self.activities[direction] = count
1965
 
 
 
2145
            _activities[direction] = count
 
2146
        self.activities = _activities
1966
2147
        # We override at class level because constructors may propagate the
1967
2148
        # bound method and render instance overriding ineffective (an
1968
2149
        # alternative would be to define a specific ui factory instead...)
1969
 
        self.orig_report_activity = self._transport._report_activity
1970
 
        self._transport._report_activity = report_activity
1971
 
 
1972
 
    def tearDown(self):
1973
 
        self._transport._report_activity = self.orig_report_activity
1974
 
        self.server.stop_server()
1975
 
        tests.TestCase.tearDown(self)
 
2150
        self.overrideAttr(self._transport, '_report_activity', report_activity)
1976
2151
 
1977
2152
    def get_transport(self):
1978
 
        return self._transport(self.server.get_url())
 
2153
        t = self._transport(self.server.get_url())
 
2154
        # FIXME: Needs cleanup -- vila 20100611
 
2155
        return t
1979
2156
 
1980
2157
    def assertActivitiesMatch(self):
1981
2158
        self.assertEqual(self.server.bytes_read,
2086
2263
'''
2087
2264
        t = self.get_transport()
2088
2265
        # We must send a single line of body bytes, see
2089
 
        # PredefinedRequestHandler.handle_one_request
 
2266
        # PredefinedRequestHandler._handle_one_request
2090
2267
        code, f = t._post('abc def end-of-body\n')
2091
2268
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2092
2269
        self.assertActivitiesMatch()
2094
2271
 
2095
2272
class TestActivity(tests.TestCase, TestActivityMixin):
2096
2273
 
 
2274
    scenarios = multiply_scenarios(
 
2275
        vary_by_http_activity(),
 
2276
        vary_by_http_protocol_version(),
 
2277
        )
 
2278
 
2097
2279
    def setUp(self):
2098
 
        tests.TestCase.setUp(self)
2099
 
        self.server = self._activity_server(self._protocol_version)
2100
 
        self.server.start_server()
2101
 
        self.activities = {}
2102
 
        def report_activity(t, bytes, direction):
2103
 
            count = self.activities.get(direction, 0)
2104
 
            count += bytes
2105
 
            self.activities[direction] = count
2106
 
 
2107
 
        # We override at class level because constructors may propagate the
2108
 
        # bound method and render instance overriding ineffective (an
2109
 
        # alternative would be to define a specific ui factory instead...)
2110
 
        self.orig_report_activity = self._transport._report_activity
2111
 
        self._transport._report_activity = report_activity
2112
 
 
2113
 
    def tearDown(self):
2114
 
        self._transport._report_activity = self.orig_report_activity
2115
 
        self.server.stop_server()
2116
 
        tests.TestCase.tearDown(self)
 
2280
        super(TestActivity, self).setUp()
 
2281
        TestActivityMixin.setUp(self)
2117
2282
 
2118
2283
 
2119
2284
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2120
2285
 
 
2286
    # Unlike TestActivity, we are really testing ReportingFileSocket and
 
2287
    # ReportingSocket, so we don't need all the parametrization. Since
 
2288
    # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
 
2289
    # test them through their use by the transport than directly (that's a
 
2290
    # bit less clean but far more simpler and effective).
 
2291
    _activity_server = ActivityHTTPServer
 
2292
    _protocol_version = 'HTTP/1.1'
 
2293
 
2121
2294
    def setUp(self):
2122
 
        tests.TestCase.setUp(self)
2123
 
        # Unlike TestActivity, we are really testing ReportingFileSocket and
2124
 
        # ReportingSocket, so we don't need all the parametrization. Since
2125
 
        # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2126
 
        # test them through their use by the transport than directly (that's a
2127
 
        # bit less clean but far more simpler and effective).
2128
 
        self.server = ActivityHTTPServer('HTTP/1.1')
2129
 
        self._transport=_urllib.HttpTransport_urllib
2130
 
 
2131
 
        self.server.start_server()
2132
 
 
2133
 
        # We override at class level because constructors may propagate the
2134
 
        # bound method and render instance overriding ineffective (an
2135
 
        # alternative would be to define a specific ui factory instead...)
2136
 
        self.orig_report_activity = self._transport._report_activity
2137
 
        self._transport._report_activity = None
2138
 
 
2139
 
    def tearDown(self):
2140
 
        self._transport._report_activity = self.orig_report_activity
2141
 
        self.server.stop_server()
2142
 
        tests.TestCase.tearDown(self)
 
2295
        super(TestNoReportActivity, self).setUp()
 
2296
        self._transport =_urllib.HttpTransport_urllib
 
2297
        TestActivityMixin.setUp(self)
2143
2298
 
2144
2299
    def assertActivitiesMatch(self):
2145
2300
        # Nothing to check here
2149
2304
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2150
2305
    """Test authentication on the redirected http server."""
2151
2306
 
 
2307
    scenarios = vary_by_http_protocol_version()
 
2308
 
2152
2309
    _auth_header = 'Authorization'
2153
2310
    _password_prompt_prefix = ''
2154
2311
    _username_prompt_prefix = ''
2155
2312
    _auth_server = http_utils.HTTPBasicAuthServer
2156
2313
    _transport = _urllib.HttpTransport_urllib
2157
2314
 
2158
 
    def create_transport_readonly_server(self):
2159
 
        return self._auth_server()
2160
 
 
2161
 
    def create_transport_secondary_server(self):
2162
 
        """Create the secondary server redirecting to the primary server"""
2163
 
        new = self.get_readonly_server()
2164
 
 
2165
 
        redirecting = http_utils.HTTPServerRedirecting()
2166
 
        redirecting.redirect_to(new.host, new.port)
2167
 
        return redirecting
2168
 
 
2169
2315
    def setUp(self):
2170
2316
        super(TestAuthOnRedirected, self).setUp()
2171
2317
        self.build_tree_contents([('a','a'),
2176
2322
                                       self.new_server.port)
2177
2323
        self.old_server.redirections = [
2178
2324
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2179
 
        self.old_transport = self._transport(self.old_server.get_url())
 
2325
        self.old_transport = self.get_old_transport()
2180
2326
        self.new_server.add_user('joe', 'foo')
2181
 
 
2182
 
    def get_a(self, transport):
2183
 
        return transport.get('a')
 
2327
        cleanup_http_redirection_connections(self)
 
2328
 
 
2329
    def create_transport_readonly_server(self):
 
2330
        server = self._auth_server(protocol_version=self._protocol_version)
 
2331
        server._url_protocol = self._url_protocol
 
2332
        return server
 
2333
 
 
2334
    def get_a(self, t):
 
2335
        return t.get('a')
2184
2336
 
2185
2337
    def test_auth_on_redirected_via_do_catching_redirections(self):
2186
2338
        self.redirections = 0
2187
2339
 
2188
 
        def redirected(transport, exception, redirection_notice):
 
2340
        def redirected(t, exception, redirection_notice):
2189
2341
            self.redirections += 1
2190
 
            dir, file = urlutils.split(exception.target)
2191
 
            return self._transport(dir)
 
2342
            redirected_t = t._redirected_to(exception.source, exception.target)
 
2343
            self.addCleanup(redirected_t.disconnect)
 
2344
            return redirected_t
2192
2345
 
2193
 
        stdout = tests.StringIOWrapper()
2194
 
        stderr = tests.StringIOWrapper()
2195
 
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2196
 
                                            stdout=stdout, stderr=stderr)
 
2346
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n')
2197
2347
        self.assertEqual('redirected once',
2198
2348
                         transport.do_catching_redirections(
2199
2349
                self.get_a, self.old_transport, redirected).read())
2201
2351
        # stdin should be empty
2202
2352
        self.assertEqual('', ui.ui_factory.stdin.readline())
2203
2353
        # stdout should be empty, stderr will contains the prompts
2204
 
        self.assertEqual('', stdout.getvalue())
 
2354
        self.assertEqual('', ui.ui_factory.stdout.getvalue())
2205
2355
 
2206
2356
    def test_auth_on_redirected_via_following_redirections(self):
2207
2357
        self.new_server.add_user('joe', 'foo')
2208
 
        stdout = tests.StringIOWrapper()
2209
 
        stderr = tests.StringIOWrapper()
2210
 
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2211
 
                                            stdout=stdout, stderr=stderr)
 
2358
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n')
2212
2359
        t = self.old_transport
2213
2360
        req = RedirectedRequest('GET', t.abspath('a'))
2214
2361
        new_prefix = 'http://%s:%s' % (self.new_server.host,
2215
2362
                                       self.new_server.port)
2216
2363
        self.old_server.redirections = [
2217
2364
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2218
 
        self.assertEqual('redirected once',t._perform(req).read())
 
2365
        self.assertEqual('redirected once', t._perform(req).read())
2219
2366
        # stdin should be empty
2220
2367
        self.assertEqual('', ui.ui_factory.stdin.readline())
2221
2368
        # stdout should be empty, stderr will contains the prompts
2222
 
        self.assertEqual('', stdout.getvalue())
2223
 
 
 
2369
        self.assertEqual('', ui.ui_factory.stdout.getvalue())
2224
2370