/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to brzlib/tests/test_http.py

  • Committer: Jelmer Vernooij
  • Date: 2017-05-21 12:41:27 UTC
  • mto: This revision was merged to the branch mainline in revision 6623.
  • Revision ID: jelmer@jelmer.uk-20170521124127-iv8etg0vwymyai6y
s/bzr/brz/ in apport config.

Show diffs side-by-side

added added

removed removed

Lines of Context:
brzlib/tests/test_http.py
1
 
# Copyright (C) 2005-2010 Canonical Ltd
 
1
# Copyright (C) 2005-2012, 2015, 2016, 2017 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
20
20
transport implementation, http protocol versions and authentication schemes.
21
21
"""
22
22
 
23
 
# TODO: Should be renamed to bzrlib.transport.http.tests?
24
 
# TODO: What about renaming to bzrlib.tests.transport.http ?
 
23
# TODO: Should be renamed to brzlib.transport.http.tests?
 
24
# TODO: What about renaming to brzlib.tests.transport.http ?
25
25
 
26
 
from cStringIO import StringIO
27
26
import httplib
28
 
import os
29
 
import select
30
27
import SimpleHTTPServer
31
28
import socket
32
29
import sys
33
30
import threading
34
31
 
35
 
import bzrlib
36
 
from bzrlib import (
37
 
    bzrdir,
 
32
import brzlib
 
33
from brzlib import (
38
34
    config,
 
35
    controldir,
 
36
    debug,
39
37
    errors,
40
38
    osutils,
41
39
    remote as _mod_remote,
42
40
    tests,
 
41
    trace,
43
42
    transport,
44
43
    ui,
45
 
    urlutils,
46
 
    )
47
 
from bzrlib.symbol_versioning import (
48
 
    deprecated_in,
49
 
    )
50
 
from bzrlib.tests import (
 
44
    )
 
45
from brzlib.tests import (
51
46
    features,
52
47
    http_server,
53
48
    http_utils,
54
 
    )
55
 
from bzrlib.transport import (
 
49
    test_server,
 
50
    )
 
51
from brzlib.tests.scenarios import (
 
52
    load_tests_apply_scenarios,
 
53
    multiply_scenarios,
 
54
    )
 
55
from brzlib.transport import (
56
56
    http,
57
57
    remote,
58
58
    )
59
 
from bzrlib.transport.http import (
 
59
from brzlib.transport.http import (
60
60
    _urllib,
61
61
    _urllib2_wrappers,
62
62
    )
63
63
 
64
64
 
65
65
if features.pycurl.available():
66
 
    from bzrlib.transport.http._pycurl import PyCurlTransport
67
 
 
68
 
 
69
 
def load_tests(standard_tests, module, loader):
70
 
    """Multiply tests for http clients and protocol versions."""
71
 
    result = loader.suiteClass()
72
 
 
73
 
    # one for each transport implementation
74
 
    t_tests, remaining_tests = tests.split_suite_by_condition(
75
 
        standard_tests, tests.condition_isinstance((
76
 
                TestHttpTransportRegistration,
77
 
                TestHttpTransportUrls,
78
 
                Test_redirected_to,
79
 
                )))
 
66
    from brzlib.transport.http._pycurl import PyCurlTransport
 
67
 
 
68
 
 
69
load_tests = load_tests_apply_scenarios
 
70
 
 
71
 
 
72
def vary_by_http_client_implementation():
 
73
    """Test the two libraries we can use, pycurl and urllib."""
80
74
    transport_scenarios = [
81
75
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
82
76
                        _server=http_server.HttpServer_urllib,
83
 
                        _qualified_prefix='http+urllib',)),
 
77
                        _url_protocol='http+urllib',)),
84
78
        ]
85
79
    if features.pycurl.available():
86
80
        transport_scenarios.append(
87
81
            ('pycurl', dict(_transport=PyCurlTransport,
88
82
                            _server=http_server.HttpServer_PyCurl,
89
 
                            _qualified_prefix='http+pycurl',)))
90
 
    tests.multiply_tests(t_tests, transport_scenarios, result)
91
 
 
92
 
    # each implementation tested with each HTTP version
93
 
    tp_tests, remaining_tests = tests.split_suite_by_condition(
94
 
        remaining_tests, tests.condition_isinstance((
95
 
                SmartHTTPTunnellingTest,
96
 
                TestDoCatchRedirections,
97
 
                TestHTTPConnections,
98
 
                TestHTTPRedirections,
99
 
                TestHTTPSilentRedirections,
100
 
                TestLimitedRangeRequestServer,
101
 
                TestPost,
102
 
                TestProxyHttpServer,
103
 
                TestRanges,
104
 
                TestSpecificRequestHandler,
105
 
                )))
106
 
    protocol_scenarios = [
107
 
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
108
 
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
109
 
            ]
110
 
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
111
 
                                            protocol_scenarios)
112
 
    tests.multiply_tests(tp_tests, tp_scenarios, result)
113
 
 
114
 
    # proxy auth: each auth scheme on all http versions on all implementations.
115
 
    tppa_tests, remaining_tests = tests.split_suite_by_condition(
116
 
        remaining_tests, tests.condition_isinstance((
117
 
                TestProxyAuth,
118
 
                )))
119
 
    proxy_auth_scheme_scenarios = [
120
 
        ('basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
121
 
        ('digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
122
 
        ('basicdigest',
123
 
         dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
83
                            _url_protocol='http+pycurl',)))
 
84
    return transport_scenarios
 
85
 
 
86
 
 
87
def vary_by_http_protocol_version():
 
88
    """Test on http/1.0 and 1.1"""
 
89
    return [
 
90
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
91
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
124
92
        ]
125
 
    tppa_scenarios = tests.multiply_scenarios(tp_scenarios,
126
 
                                              proxy_auth_scheme_scenarios)
127
 
    tests.multiply_tests(tppa_tests, tppa_scenarios, result)
128
 
 
129
 
    # auth: each auth scheme on all http versions on all implementations.
130
 
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
131
 
        remaining_tests, tests.condition_isinstance((
132
 
                TestAuth,
133
 
                )))
134
 
    auth_scheme_scenarios = [
 
93
 
 
94
 
 
95
def vary_by_http_auth_scheme():
 
96
    scenarios = [
135
97
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
136
98
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
137
99
        ('basicdigest',
138
 
         dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
139
 
        ]
140
 
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
141
 
                                             auth_scheme_scenarios)
142
 
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
143
 
 
144
 
    # activity: on all http[s] versions on all implementations
145
 
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
146
 
        remaining_tests, tests.condition_isinstance((
147
 
                TestActivity,
148
 
                )))
 
100
            dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
 
101
        ]
 
102
    # Add some attributes common to all scenarios
 
103
    for scenario_id, scenario_dict in scenarios:
 
104
        scenario_dict.update(_auth_header='Authorization',
 
105
                             _username_prompt_prefix='',
 
106
                             _password_prompt_prefix='')
 
107
    return scenarios
 
108
 
 
109
 
 
110
def vary_by_http_proxy_auth_scheme():
 
111
    scenarios = [
 
112
        ('proxy-basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
 
113
        ('proxy-digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
 
114
        ('proxy-basicdigest',
 
115
            dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
116
        ]
 
117
    # Add some attributes common to all scenarios
 
118
    for scenario_id, scenario_dict in scenarios:
 
119
        scenario_dict.update(_auth_header='Proxy-Authorization',
 
120
                             _username_prompt_prefix='Proxy ',
 
121
                             _password_prompt_prefix='Proxy ')
 
122
    return scenarios
 
123
 
 
124
 
 
125
def vary_by_http_activity():
149
126
    activity_scenarios = [
150
127
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
151
 
                             _transport=_urllib.HttpTransport_urllib,)),
 
128
                            _transport=_urllib.HttpTransport_urllib,)),
152
129
        ]
153
 
    if tests.HTTPSServerFeature.available():
 
130
    if features.pycurl.available():
 
131
        activity_scenarios.append(
 
132
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
 
133
                                _transport=PyCurlTransport,)),)
 
134
    if features.HTTPSServerFeature.available():
 
135
        # FIXME: Until we have a better way to handle self-signed certificates
 
136
        # (like allowing them in a test specific authentication.conf for
 
137
        # example), we need some specialized pycurl/urllib transport for tests.
 
138
        # -- vila 2012-01-20
 
139
        from brzlib.tests import (
 
140
            ssl_certs,
 
141
            )
 
142
        class HTTPS_urllib_transport(_urllib.HttpTransport_urllib):
 
143
 
 
144
            def __init__(self, base, _from_transport=None):
 
145
                super(HTTPS_urllib_transport, self).__init__(
 
146
                    base, _from_transport=_from_transport,
 
147
                    ca_certs=ssl_certs.build_path('ca.crt'))
 
148
 
154
149
        activity_scenarios.append(
155
150
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
156
 
                                  _transport=_urllib.HttpTransport_urllib,)),)
157
 
    if features.pycurl.available():
158
 
        activity_scenarios.append(
159
 
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
160
 
                                 _transport=PyCurlTransport,)),)
161
 
        if tests.HTTPSServerFeature.available():
162
 
            from bzrlib.tests import (
163
 
                ssl_certs,
164
 
                )
165
 
            # FIXME: Until we have a better way to handle self-signed
166
 
            # certificates (like allowing them in a test specific
167
 
            # authentication.conf for example), we need some specialized pycurl
168
 
            # transport for tests.
 
151
                                  _transport=HTTPS_urllib_transport,)),)
 
152
        if features.pycurl.available():
169
153
            class HTTPS_pycurl_transport(PyCurlTransport):
170
154
 
171
155
                def __init__(self, base, _from_transport=None):
175
159
 
176
160
            activity_scenarios.append(
177
161
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
178
 
                                      _transport=HTTPS_pycurl_transport,)),)
179
 
 
180
 
    tpact_scenarios = tests.multiply_scenarios(activity_scenarios,
181
 
                                               protocol_scenarios)
182
 
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
183
 
 
184
 
    # No parametrization for the remaining tests
185
 
    result.addTests(remaining_tests)
186
 
 
187
 
    return result
 
162
                                    _transport=HTTPS_pycurl_transport,)),)
 
163
    return activity_scenarios
188
164
 
189
165
 
190
166
class FakeManager(object):
223
199
        self._sock.bind(('127.0.0.1', 0))
224
200
        self.host, self.port = self._sock.getsockname()
225
201
        self._ready = threading.Event()
226
 
        self._thread = threading.Thread(target=self._accept_read_and_reply)
227
 
        self._thread.setDaemon(True)
 
202
        self._thread = test_server.TestThread(
 
203
            sync_event=self._ready, target=self._accept_read_and_reply)
228
204
        self._thread.start()
229
 
        self._ready.wait(5)
 
205
        if 'threads' in tests.selftest_debug_flags:
 
206
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
 
207
        self._ready.wait()
230
208
 
231
209
    def _accept_read_and_reply(self):
232
210
        self._sock.listen(1)
233
211
        self._ready.set()
234
 
        self._sock.settimeout(5)
235
 
        try:
236
 
            conn, address = self._sock.accept()
237
 
            # On win32, the accepted connection will be non-blocking to start
238
 
            # with because we're using settimeout.
239
 
            conn.setblocking(True)
 
212
        conn, address = self._sock.accept()
 
213
        if self._expect_body_tail is not None:
240
214
            while not self.received_bytes.endswith(self._expect_body_tail):
241
215
                self.received_bytes += conn.recv(4096)
242
216
            conn.sendall('HTTP/1.1 200 OK\r\n')
243
 
        except socket.timeout:
244
 
            # Make sure the client isn't stuck waiting for us to e.g. accept.
 
217
        try:
245
218
            self._sock.close()
246
219
        except socket.error:
247
220
            # The client may have already closed the socket.
249
222
 
250
223
    def stop_server(self):
251
224
        try:
252
 
            self._sock.close()
 
225
            # Issue a fake connection to wake up the server and allow it to
 
226
            # finish quickly
 
227
            fake_conn = osutils.connect_socket((self.host, self.port))
 
228
            fake_conn.close()
253
229
        except socket.error:
254
230
            # We might have already closed it.  We don't care.
255
231
            pass
256
232
        self.host = None
257
233
        self.port = None
 
234
        self._thread.join()
 
235
        if 'threads' in tests.selftest_debug_flags:
 
236
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
258
237
 
259
238
 
260
239
class TestAuthHeader(tests.TestCase):
281
260
        self.assertEqual('basic', scheme)
282
261
        self.assertEqual('realm="Thou should not pass"', remainder)
283
262
 
 
263
    def test_build_basic_header_with_long_creds(self):
 
264
        handler = _urllib2_wrappers.BasicAuthHandler()
 
265
        user = 'user' * 10  # length 40
 
266
        password = 'password' * 5  # length 40
 
267
        header = handler.build_auth_header(
 
268
            dict(user=user, password=password), None)
 
269
        # https://bugs.launchpad.net/bzr/+bug/1606203 was caused by incorrectly
 
270
        # creating a header value with an embedded '\n'
 
271
        self.assertFalse('\n' in header)
 
272
 
284
273
    def test_basic_extract_realm(self):
285
274
        scheme, remainder = self.parse_header(
286
275
            'Basic realm="Thou should not pass"',
296
285
        self.assertEqual('realm="Thou should not pass"', remainder)
297
286
 
298
287
 
 
288
class TestHTTPRangeParsing(tests.TestCase):
 
289
 
 
290
    def setUp(self):
 
291
        super(TestHTTPRangeParsing, self).setUp()
 
292
        # We focus on range  parsing here and ignore everything else
 
293
        class RequestHandler(http_server.TestingHTTPRequestHandler):
 
294
            def setup(self): pass
 
295
            def handle(self): pass
 
296
            def finish(self): pass
 
297
 
 
298
        self.req_handler = RequestHandler(None, None, None)
 
299
 
 
300
    def assertRanges(self, ranges, header, file_size):
 
301
        self.assertEqual(ranges,
 
302
                          self.req_handler._parse_ranges(header, file_size))
 
303
 
 
304
    def test_simple_range(self):
 
305
        self.assertRanges([(0,2)], 'bytes=0-2', 12)
 
306
 
 
307
    def test_tail(self):
 
308
        self.assertRanges([(8, 11)], 'bytes=-4', 12)
 
309
 
 
310
    def test_tail_bigger_than_file(self):
 
311
        self.assertRanges([(0, 11)], 'bytes=-99', 12)
 
312
 
 
313
    def test_range_without_end(self):
 
314
        self.assertRanges([(4, 11)], 'bytes=4-', 12)
 
315
 
 
316
    def test_invalid_ranges(self):
 
317
        self.assertRanges(None, 'bytes=12-22', 12)
 
318
        self.assertRanges(None, 'bytes=1-3,12-22', 12)
 
319
        self.assertRanges(None, 'bytes=-', 12)
 
320
 
 
321
 
299
322
class TestHTTPServer(tests.TestCase):
300
323
    """Test the HTTP servers implementations."""
301
324
 
304
327
 
305
328
            protocol_version = 'HTTP/0.1'
306
329
 
307
 
        server = http_server.HttpServer(BogusRequestHandler)
308
 
        try:
309
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
310
 
        except:
311
 
            server.stop_server()
312
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
330
        self.assertRaises(httplib.UnknownProtocol,
 
331
                          http_server.HttpServer, BogusRequestHandler)
313
332
 
314
333
    def test_force_invalid_protocol(self):
315
 
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
316
 
        try:
317
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
318
 
        except:
319
 
            server.stop_server()
320
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
334
        self.assertRaises(httplib.UnknownProtocol,
 
335
                          http_server.HttpServer, protocol_version='HTTP/0.1')
321
336
 
322
337
    def test_server_start_and_stop(self):
323
338
        server = http_server.HttpServer()
 
339
        self.addCleanup(server.stop_server)
324
340
        server.start_server()
325
 
        try:
326
 
            self.assertTrue(server._http_running)
327
 
        finally:
328
 
            server.stop_server()
329
 
        self.assertFalse(server._http_running)
 
341
        self.assertTrue(server.server is not None)
 
342
        self.assertTrue(server.server.serving is not None)
 
343
        self.assertTrue(server.server.serving)
330
344
 
331
345
    def test_create_http_server_one_zero(self):
332
346
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
335
349
 
336
350
        server = http_server.HttpServer(RequestHandlerOneZero)
337
351
        self.start_server(server)
338
 
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
 
352
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
339
353
 
340
354
    def test_create_http_server_one_one(self):
341
355
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
344
358
 
345
359
        server = http_server.HttpServer(RequestHandlerOneOne)
346
360
        self.start_server(server)
347
 
        self.assertIsInstance(server._httpd,
 
361
        self.assertIsInstance(server.server,
348
362
                              http_server.TestingThreadingHTTPServer)
349
363
 
350
364
    def test_create_http_server_force_one_one(self):
355
369
        server = http_server.HttpServer(RequestHandlerOneZero,
356
370
                                        protocol_version='HTTP/1.1')
357
371
        self.start_server(server)
358
 
        self.assertIsInstance(server._httpd,
 
372
        self.assertIsInstance(server.server,
359
373
                              http_server.TestingThreadingHTTPServer)
360
374
 
361
375
    def test_create_http_server_force_one_zero(self):
366
380
        server = http_server.HttpServer(RequestHandlerOneOne,
367
381
                                        protocol_version='HTTP/1.0')
368
382
        self.start_server(server)
369
 
        self.assertIsInstance(server._httpd,
 
383
        self.assertIsInstance(server.server,
370
384
                              http_server.TestingHTTPServer)
371
385
 
372
386
 
380
394
    _transport = property(_get_pycurl_maybe)
381
395
 
382
396
 
383
 
class TestHttpUrls(tests.TestCase):
384
 
 
385
 
    # TODO: This should be moved to authorization tests once they
386
 
    # are written.
387
 
 
388
 
    def test_url_parsing(self):
389
 
        f = FakeManager()
390
 
        url = http.extract_auth('http://example.com', f)
391
 
        self.assertEqual('http://example.com', url)
392
 
        self.assertEqual(0, len(f.credentials))
393
 
        url = http.extract_auth(
394
 
            'http://user:pass@example.com/bzr/bzr.dev', f)
395
 
        self.assertEqual('http://example.com/bzr/bzr.dev', url)
396
 
        self.assertEqual(1, len(f.credentials))
397
 
        self.assertEqual([None, 'example.com', 'user', 'pass'],
398
 
                         f.credentials[0])
399
 
 
400
 
 
401
397
class TestHttpTransportUrls(tests.TestCase):
402
398
    """Test the http urls."""
403
399
 
 
400
    scenarios = vary_by_http_client_implementation()
 
401
 
404
402
    def test_abs_url(self):
405
403
        """Construction of absolute http URLs"""
406
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
404
        t = self._transport('http://example.com/bzr/bzr.dev/')
407
405
        eq = self.assertEqualDiff
408
 
        eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
409
 
        eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
410
 
        eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
 
406
        eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
 
407
        eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
 
408
        eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
411
409
        eq(t.abspath('.bzr/1//2/./3'),
412
 
           'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
 
410
           'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
413
411
 
414
412
    def test_invalid_http_urls(self):
415
413
        """Trap invalid construction of urls"""
416
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
414
        self._transport('http://example.com/bzr/bzr.dev/')
417
415
        self.assertRaises(errors.InvalidURL,
418
416
                          self._transport,
419
 
                          'http://http://bazaar-vcs.org/bzr/bzr.dev/')
 
417
                          'http://http://example.com/bzr/bzr.dev/')
420
418
 
421
419
    def test_http_root_urls(self):
422
420
        """Construction of URLs from server root"""
423
 
        t = self._transport('http://bzr.ozlabs.org/')
 
421
        t = self._transport('http://example.com/')
424
422
        eq = self.assertEqualDiff
425
423
        eq(t.abspath('.bzr/tree-version'),
426
 
           'http://bzr.ozlabs.org/.bzr/tree-version')
 
424
           'http://example.com/.bzr/tree-version')
427
425
 
428
426
    def test_http_impl_urls(self):
429
427
        """There are servers which ask for particular clients to connect"""
431
429
        server.start_server()
432
430
        try:
433
431
            url = server.get_url()
434
 
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
 
432
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
435
433
        finally:
436
434
            server.stop_server()
437
435
 
475
473
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
476
474
    """Test the http connections."""
477
475
 
 
476
    scenarios = multiply_scenarios(
 
477
        vary_by_http_client_implementation(),
 
478
        vary_by_http_protocol_version(),
 
479
        )
 
480
 
478
481
    def setUp(self):
479
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
482
        super(TestHTTPConnections, self).setUp()
480
483
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
481
484
                        transport=self.get_transport())
482
485
 
483
486
    def test_http_has(self):
484
487
        server = self.get_readonly_server()
485
 
        t = self._transport(server.get_url())
 
488
        t = self.get_readonly_transport()
486
489
        self.assertEqual(t.has('foo/bar'), True)
487
490
        self.assertEqual(len(server.logs), 1)
488
491
        self.assertContainsRe(server.logs[0],
490
493
 
491
494
    def test_http_has_not_found(self):
492
495
        server = self.get_readonly_server()
493
 
        t = self._transport(server.get_url())
 
496
        t = self.get_readonly_transport()
494
497
        self.assertEqual(t.has('not-found'), False)
495
498
        self.assertContainsRe(server.logs[1],
496
499
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
497
500
 
498
501
    def test_http_get(self):
499
502
        server = self.get_readonly_server()
500
 
        t = self._transport(server.get_url())
 
503
        t = self.get_readonly_transport()
501
504
        fp = t.get('foo/bar')
502
505
        self.assertEqualDiff(
503
506
            fp.read(),
505
508
        self.assertEqual(len(server.logs), 1)
506
509
        self.assertTrue(server.logs[0].find(
507
510
            '"GET /foo/bar HTTP/1.1" 200 - "-" "bzr/%s'
508
 
            % bzrlib.__version__) > -1)
 
511
            % brzlib.__version__) > -1)
509
512
 
510
513
    def test_has_on_bogus_host(self):
511
514
        # Get a free address and don't 'accept' on it, so that we
525
528
class TestHttpTransportRegistration(tests.TestCase):
526
529
    """Test registrations of various http implementations"""
527
530
 
 
531
    scenarios = vary_by_http_client_implementation()
 
532
 
528
533
    def test_http_registered(self):
529
 
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
 
534
        t = transport.get_transport_from_url(
 
535
            '%s://foo.com/' % self._url_protocol)
530
536
        self.assertIsInstance(t, transport.Transport)
531
537
        self.assertIsInstance(t, self._transport)
532
538
 
533
539
 
534
540
class TestPost(tests.TestCase):
535
541
 
 
542
    scenarios = multiply_scenarios(
 
543
        vary_by_http_client_implementation(),
 
544
        vary_by_http_protocol_version(),
 
545
        )
 
546
 
536
547
    def test_post_body_is_received(self):
537
548
        server = RecordingServer(expect_body_tail='end-of-body',
538
 
            scheme=self._qualified_prefix)
 
549
                                 scheme=self._url_protocol)
539
550
        self.start_server(server)
540
551
        url = server.get_url()
541
 
        http_transport = self._transport(url)
 
552
        # FIXME: needs a cleanup -- vila 20100611
 
553
        http_transport = transport.get_transport_from_url(url)
542
554
        code, response = http_transport._post('abc def end-of-body')
543
555
        self.assertTrue(
544
556
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
545
557
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
 
558
        self.assertTrue('content-type: application/octet-stream\r'
 
559
                        in server.received_bytes.lower())
546
560
        # The transport should not be assuming that the server can accept
547
561
        # chunked encoding the first time it connects, because HTTP/1.1, so we
548
562
        # check for the literal string.
584
598
    Daughter classes are expected to override _req_handler_class
585
599
    """
586
600
 
 
601
    scenarios = multiply_scenarios(
 
602
        vary_by_http_client_implementation(),
 
603
        vary_by_http_protocol_version(),
 
604
        )
 
605
 
587
606
    # Provide a useful default
588
607
    _req_handler_class = http_server.TestingHTTPRequestHandler
589
608
 
590
609
    def create_transport_readonly_server(self):
591
 
        return http_server.HttpServer(self._req_handler_class,
592
 
                                      protocol_version=self._protocol_version)
 
610
        server = http_server.HttpServer(self._req_handler_class,
 
611
                                        protocol_version=self._protocol_version)
 
612
        server._url_protocol = self._url_protocol
 
613
        return server
593
614
 
594
615
    def _testing_pycurl(self):
595
616
        # TODO: This is duplicated for lots of the classes in this file
600
621
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
601
622
    """Whatever request comes in, close the connection"""
602
623
 
603
 
    def handle_one_request(self):
 
624
    def _handle_one_request(self):
604
625
        """Handle a single HTTP request, by abruptly closing the connection"""
605
626
        self.close_connection = 1
606
627
 
611
632
    _req_handler_class = WallRequestHandler
612
633
 
613
634
    def test_http_has(self):
614
 
        server = self.get_readonly_server()
615
 
        t = self._transport(server.get_url())
 
635
        t = self.get_readonly_transport()
616
636
        # Unfortunately httplib (see HTTPResponse._read_status
617
637
        # for details) make no distinction between a closed
618
638
        # socket and badly formatted status line, so we can't
624
644
                          t.has, 'foo/bar')
625
645
 
626
646
    def test_http_get(self):
627
 
        server = self.get_readonly_server()
628
 
        t = self._transport(server.get_url())
 
647
        t = self.get_readonly_transport()
629
648
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
630
649
                           errors.InvalidHttpResponse),
631
650
                          t.get, 'foo/bar')
647
666
 
648
667
    _req_handler_class = BadStatusRequestHandler
649
668
 
 
669
    def setUp(self):
 
670
        super(TestBadStatusServer, self).setUp()
 
671
        # See https://bugs.launchpad.net/bzr/+bug/1451448 for details.
 
672
        # TD;LR: Running both a TCP client and server in the same process and
 
673
        # thread uncovers a race in python. The fix is to run the server in a
 
674
        # different process. Trying to fix yet another race here is not worth
 
675
        # the effort. -- vila 2015-09-06
 
676
        if 'HTTP/1.0' in self.id():
 
677
            raise tests.TestSkipped(
 
678
                'Client/Server in the same process and thread can hang')
 
679
 
650
680
    def test_http_has(self):
651
 
        server = self.get_readonly_server()
652
 
        t = self._transport(server.get_url())
653
 
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
 
681
        t = self.get_readonly_transport()
 
682
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
683
                           errors.InvalidHttpResponse),
 
684
                          t.has, 'foo/bar')
654
685
 
655
686
    def test_http_get(self):
656
 
        server = self.get_readonly_server()
657
 
        t = self._transport(server.get_url())
658
 
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
 
687
        t = self.get_readonly_transport()
 
688
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
689
                           errors.InvalidHttpResponse),
 
690
                          t.get, 'foo/bar')
659
691
 
660
692
 
661
693
class InvalidStatusRequestHandler(http_server.TestingHTTPRequestHandler):
665
697
        """Fakes handling a single HTTP request, returns a bad status"""
666
698
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
667
699
        self.wfile.write("Invalid status line\r\n")
 
700
        # If we don't close the connection pycurl will hang. Since this is a
 
701
        # stress test we don't *have* to respect the protocol, but we don't
 
702
        # have to sabotage it too much either.
 
703
        self.close_connection = True
668
704
        return False
669
705
 
670
706
 
676
712
 
677
713
    _req_handler_class = InvalidStatusRequestHandler
678
714
 
679
 
    def test_http_has(self):
680
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
681
 
            raise tests.KnownFailure(
682
 
                'pycurl hangs if the server send back garbage')
683
 
        super(TestInvalidStatusServer, self).test_http_has()
684
 
 
685
 
    def test_http_get(self):
686
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
687
 
            raise tests.KnownFailure(
688
 
                'pycurl hangs if the server send back garbage')
689
 
        super(TestInvalidStatusServer, self).test_http_get()
690
 
 
691
715
 
692
716
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
693
717
    """Whatever request comes in, returns a bad protocol version"""
715
739
        super(TestBadProtocolServer, self).setUp()
716
740
 
717
741
    def test_http_has(self):
718
 
        server = self.get_readonly_server()
719
 
        t = self._transport(server.get_url())
 
742
        t = self.get_readonly_transport()
720
743
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
721
744
 
722
745
    def test_http_get(self):
723
 
        server = self.get_readonly_server()
724
 
        t = self._transport(server.get_url())
 
746
        t = self.get_readonly_transport()
725
747
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
726
748
 
727
749
 
741
763
    _req_handler_class = ForbiddenRequestHandler
742
764
 
743
765
    def test_http_has(self):
744
 
        server = self.get_readonly_server()
745
 
        t = self._transport(server.get_url())
 
766
        t = self.get_readonly_transport()
746
767
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
747
768
 
748
769
    def test_http_get(self):
749
 
        server = self.get_readonly_server()
750
 
        t = self._transport(server.get_url())
 
770
        t = self.get_readonly_transport()
751
771
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
752
772
 
753
773
 
792
812
        self.build_tree_contents([('a', '0123456789')],)
793
813
 
794
814
    def test_readv(self):
795
 
        server = self.get_readonly_server()
796
 
        t = self._transport(server.get_url())
 
815
        t = self.get_readonly_transport()
797
816
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
798
817
        self.assertEqual(l[0], (0, '0'))
799
818
        self.assertEqual(l[1], (1, '1'))
801
820
        self.assertEqual(l[3], (9, '9'))
802
821
 
803
822
    def test_readv_out_of_order(self):
804
 
        server = self.get_readonly_server()
805
 
        t = self._transport(server.get_url())
 
823
        t = self.get_readonly_transport()
806
824
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
807
825
        self.assertEqual(l[0], (1, '1'))
808
826
        self.assertEqual(l[1], (9, '9'))
810
828
        self.assertEqual(l[3], (3, '34'))
811
829
 
812
830
    def test_readv_invalid_ranges(self):
813
 
        server = self.get_readonly_server()
814
 
        t = self._transport(server.get_url())
 
831
        t = self.get_readonly_transport()
815
832
 
816
833
        # This is intentionally reading off the end of the file
817
834
        # since we are sure that it cannot get there
825
842
 
826
843
    def test_readv_multiple_get_requests(self):
827
844
        server = self.get_readonly_server()
828
 
        t = self._transport(server.get_url())
 
845
        t = self.get_readonly_transport()
829
846
        # force transport to issue multiple requests
830
847
        t._max_readv_combine = 1
831
848
        t._max_get_ranges = 1
839
856
 
840
857
    def test_readv_get_max_size(self):
841
858
        server = self.get_readonly_server()
842
 
        t = self._transport(server.get_url())
 
859
        t = self.get_readonly_transport()
843
860
        # force transport to issue multiple requests by limiting the number of
844
861
        # bytes by request. Note that this apply to coalesced offsets only, a
845
862
        # single range will keep its size even if bigger than the limit.
854
871
 
855
872
    def test_complete_readv_leave_pipe_clean(self):
856
873
        server = self.get_readonly_server()
857
 
        t = self._transport(server.get_url())
 
874
        t = self.get_readonly_transport()
858
875
        # force transport to issue multiple requests
859
876
        t._get_max_size = 2
860
 
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
 
877
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
861
878
        # The server should have issued 3 requests
862
879
        self.assertEqual(3, server.GET_request_nb)
863
880
        self.assertEqual('0123456789', t.get_bytes('a'))
865
882
 
866
883
    def test_incomplete_readv_leave_pipe_clean(self):
867
884
        server = self.get_readonly_server()
868
 
        t = self._transport(server.get_url())
 
885
        t = self.get_readonly_transport()
869
886
        # force transport to issue multiple requests
870
887
        t._get_max_size = 2
871
888
        # Don't collapse readv results into a list so that we leave unread
940
957
    def get_multiple_ranges(self, file, file_size, ranges):
941
958
        self.send_response(206)
942
959
        self.send_header('Accept-Ranges', 'bytes')
 
960
        # XXX: this is strange; the 'random' name below seems undefined and
 
961
        # yet the tests pass -- mbp 2010-10-11 bug 658773
943
962
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
944
963
        self.send_header("Content-Type",
945
964
                         "multipart/byteranges; boundary=%s" % boundary)
1007
1026
                return
1008
1027
            self.send_range_content(file, start, end - start + 1)
1009
1028
            cur += 1
1010
 
        # No final boundary
 
1029
        # Final boundary
1011
1030
        self.wfile.write(boundary_line)
1012
1031
 
1013
1032
 
1021
1040
 
1022
1041
    def test_readv_with_short_reads(self):
1023
1042
        server = self.get_readonly_server()
1024
 
        t = self._transport(server.get_url())
 
1043
        t = self.get_readonly_transport()
1025
1044
        # Force separate ranges for each offset
1026
1045
        t._bytes_to_read_before_seek = 0
1027
1046
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1042
1061
        # that mode
1043
1062
        self.assertEqual('single', t._range_hint)
1044
1063
 
 
1064
 
 
1065
class TruncatedBeforeBoundaryRequestHandler(
 
1066
    http_server.TestingHTTPRequestHandler):
 
1067
    """Truncation before a boundary, like in bug 198646"""
 
1068
 
 
1069
    _truncated_ranges = 1
 
1070
 
 
1071
    def get_multiple_ranges(self, file, file_size, ranges):
 
1072
        self.send_response(206)
 
1073
        self.send_header('Accept-Ranges', 'bytes')
 
1074
        boundary = 'tagada'
 
1075
        self.send_header('Content-Type',
 
1076
                         'multipart/byteranges; boundary=%s' % boundary)
 
1077
        boundary_line = '--%s\r\n' % boundary
 
1078
        # Calculate the Content-Length
 
1079
        content_length = 0
 
1080
        for (start, end) in ranges:
 
1081
            content_length += len(boundary_line)
 
1082
            content_length += self._header_line_length(
 
1083
                'Content-type', 'application/octet-stream')
 
1084
            content_length += self._header_line_length(
 
1085
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
 
1086
            content_length += len('\r\n') # end headers
 
1087
            content_length += end - start # + 1
 
1088
        content_length += len(boundary_line)
 
1089
        self.send_header('Content-length', content_length)
 
1090
        self.end_headers()
 
1091
 
 
1092
        # Send the multipart body
 
1093
        cur = 0
 
1094
        for (start, end) in ranges:
 
1095
            if cur + self._truncated_ranges >= len(ranges):
 
1096
                # Abruptly ends the response and close the connection
 
1097
                self.close_connection = 1
 
1098
                return
 
1099
            self.wfile.write(boundary_line)
 
1100
            self.send_header('Content-type', 'application/octet-stream')
 
1101
            self.send_header('Content-Range', 'bytes %d-%d/%d'
 
1102
                             % (start, end, file_size))
 
1103
            self.end_headers()
 
1104
            self.send_range_content(file, start, end - start + 1)
 
1105
            cur += 1
 
1106
        # Final boundary
 
1107
        self.wfile.write(boundary_line)
 
1108
 
 
1109
 
 
1110
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
 
1111
    """Tests the case of bug 198646, disconnecting before a boundary."""
 
1112
 
 
1113
    _req_handler_class = TruncatedBeforeBoundaryRequestHandler
 
1114
 
 
1115
    def setUp(self):
 
1116
        super(TestTruncatedBeforeBoundary, self).setUp()
 
1117
        self.build_tree_contents([('a', '0123456789')],)
 
1118
 
 
1119
    def test_readv_with_short_reads(self):
 
1120
        server = self.get_readonly_server()
 
1121
        t = self.get_readonly_transport()
 
1122
        # Force separate ranges for each offset
 
1123
        t._bytes_to_read_before_seek = 0
 
1124
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
 
1125
        self.assertEqual((0, '0'), ireadv.next())
 
1126
        self.assertEqual((2, '2'), ireadv.next())
 
1127
        self.assertEqual((4, '45'), ireadv.next())
 
1128
        self.assertEqual((9, '9'), ireadv.next())
 
1129
 
 
1130
 
1045
1131
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1046
1132
    """Errors out when range specifiers exceed the limit"""
1047
1133
 
1071
1157
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1072
1158
    """Tests readv requests against a server erroring out on too much ranges."""
1073
1159
 
 
1160
    scenarios = multiply_scenarios(
 
1161
        vary_by_http_client_implementation(),
 
1162
        vary_by_http_protocol_version(),
 
1163
        )
 
1164
 
1074
1165
    # Requests with more range specifiers will error out
1075
1166
    range_limit = 3
1076
1167
 
1078
1169
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1079
1170
                                      protocol_version=self._protocol_version)
1080
1171
 
1081
 
    def get_transport(self):
1082
 
        return self._transport(self.get_readonly_server().get_url())
1083
 
 
1084
1172
    def setUp(self):
1085
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1173
        super(TestLimitedRangeRequestServer, self).setUp()
1086
1174
        # We need to manipulate ranges that correspond to real chunks in the
1087
1175
        # response, so we build a content appropriately.
1088
1176
        filler = ''.join(['abcdefghij' for x in range(102)])
1090
1178
        self.build_tree_contents([('a', content)],)
1091
1179
 
1092
1180
    def test_few_ranges(self):
1093
 
        t = self.get_transport()
 
1181
        t = self.get_readonly_transport()
1094
1182
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1095
1183
        self.assertEqual(l[0], (0, '0000'))
1096
1184
        self.assertEqual(l[1], (1024, '0001'))
1097
1185
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1098
1186
 
1099
1187
    def test_more_ranges(self):
1100
 
        t = self.get_transport()
 
1188
        t = self.get_readonly_transport()
1101
1189
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1102
1190
        self.assertEqual(l[0], (0, '0000'))
1103
1191
        self.assertEqual(l[1], (1024, '0001'))
1114
1202
    Only the urllib implementation is tested here.
1115
1203
    """
1116
1204
 
1117
 
    def setUp(self):
1118
 
        tests.TestCase.setUp(self)
1119
 
        self._old_env = {}
1120
 
 
1121
 
    def tearDown(self):
1122
 
        self._restore_env()
1123
 
        tests.TestCase.tearDown(self)
1124
 
 
1125
 
    def _install_env(self, env):
1126
 
        for name, value in env.iteritems():
1127
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1128
 
 
1129
 
    def _restore_env(self):
1130
 
        for name, value in self._old_env.iteritems():
1131
 
            osutils.set_or_unset_env(name, value)
1132
 
 
1133
1205
    def _proxied_request(self):
1134
1206
        handler = _urllib2_wrappers.ProxyHandler()
1135
 
        request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
 
1207
        request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
1136
1208
        handler.set_proxy(request, 'http')
1137
1209
        return request
1138
1210
 
 
1211
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
 
1212
        handler = _urllib2_wrappers.ProxyHandler()
 
1213
        self.assertEqual(expected,
 
1214
                          handler.evaluate_proxy_bypass(host, no_proxy))
 
1215
 
1139
1216
    def test_empty_user(self):
1140
 
        self._install_env({'http_proxy': 'http://bar.com'})
 
1217
        self.overrideEnv('http_proxy', 'http://bar.com')
 
1218
        request = self._proxied_request()
 
1219
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
 
1220
 
 
1221
    def test_user_with_at(self):
 
1222
        self.overrideEnv('http_proxy',
 
1223
                         'http://username@domain:password@proxy_host:1234')
1141
1224
        request = self._proxied_request()
1142
1225
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1143
1226
 
1144
1227
    def test_invalid_proxy(self):
1145
1228
        """A proxy env variable without scheme"""
1146
 
        self._install_env({'http_proxy': 'host:1234'})
 
1229
        self.overrideEnv('http_proxy', 'host:1234')
1147
1230
        self.assertRaises(errors.InvalidURL, self._proxied_request)
1148
1231
 
 
1232
    def test_evaluate_proxy_bypass_true(self):
 
1233
        """The host is not proxied"""
 
1234
        self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
 
1235
        self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
 
1236
 
 
1237
    def test_evaluate_proxy_bypass_false(self):
 
1238
        """The host is proxied"""
 
1239
        self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
 
1240
 
 
1241
    def test_evaluate_proxy_bypass_unknown(self):
 
1242
        """The host is not explicitly proxied"""
 
1243
        self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
 
1244
        self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
 
1245
 
 
1246
    def test_evaluate_proxy_bypass_empty_entries(self):
 
1247
        """Ignore empty entries"""
 
1248
        self.assertEvaluateProxyBypass(None, 'example.com', '')
 
1249
        self.assertEvaluateProxyBypass(None, 'example.com', ',')
 
1250
        self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
 
1251
 
1149
1252
 
1150
1253
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
1151
1254
    """Tests proxy server.
1156
1259
    to the file names).
1157
1260
    """
1158
1261
 
 
1262
    scenarios = multiply_scenarios(
 
1263
        vary_by_http_client_implementation(),
 
1264
        vary_by_http_protocol_version(),
 
1265
        )
 
1266
 
1159
1267
    # FIXME: We don't have an https server available, so we don't
1160
 
    # test https connections.
 
1268
    # test https connections. --vila toolongago
1161
1269
 
1162
1270
    def setUp(self):
1163
1271
        super(TestProxyHttpServer, self).setUp()
 
1272
        self.transport_secondary_server = http_utils.ProxyServer
1164
1273
        self.build_tree_contents([('foo', 'contents of foo\n'),
1165
1274
                                  ('foo-proxied', 'proxied contents of foo\n')])
1166
1275
        # Let's setup some attributes for tests
1167
 
        self.server = self.get_readonly_server()
1168
 
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
 
1276
        server = self.get_readonly_server()
 
1277
        self.server_host_port = '%s:%d' % (server.host, server.port)
1169
1278
        if self._testing_pycurl():
1170
1279
            # Oh my ! pycurl does not check for the port as part of
1171
1280
            # no_proxy :-( So we just test the host part
1172
 
            self.no_proxy_host = self.server.host
 
1281
            self.no_proxy_host = server.host
1173
1282
        else:
1174
 
            self.no_proxy_host = self.proxy_address
 
1283
            self.no_proxy_host = self.server_host_port
1175
1284
        # The secondary server is the proxy
1176
 
        self.proxy = self.get_secondary_server()
1177
 
        self.proxy_url = self.proxy.get_url()
1178
 
        self._old_env = {}
 
1285
        self.proxy_url = self.get_secondary_url()
 
1286
        if self._testing_pycurl():
 
1287
            self.proxy_url = self.proxy_url.replace('+pycurl', '')
1179
1288
 
1180
1289
    def _testing_pycurl(self):
1181
1290
        # TODO: This is duplicated for lots of the classes in this file
1182
1291
        return (features.pycurl.available()
1183
1292
                and self._transport == PyCurlTransport)
1184
1293
 
1185
 
    def create_transport_secondary_server(self):
1186
 
        """Creates an http server that will serve files with
1187
 
        '-proxied' appended to their names.
1188
 
        """
1189
 
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
1190
 
 
1191
 
    def _install_env(self, env):
1192
 
        for name, value in env.iteritems():
1193
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1194
 
 
1195
 
    def _restore_env(self):
1196
 
        for name, value in self._old_env.iteritems():
1197
 
            osutils.set_or_unset_env(name, value)
1198
 
 
1199
 
    def proxied_in_env(self, env):
1200
 
        self._install_env(env)
1201
 
        url = self.server.get_url()
1202
 
        t = self._transport(url)
1203
 
        try:
1204
 
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1205
 
        finally:
1206
 
            self._restore_env()
1207
 
 
1208
 
    def not_proxied_in_env(self, env):
1209
 
        self._install_env(env)
1210
 
        url = self.server.get_url()
1211
 
        t = self._transport(url)
1212
 
        try:
1213
 
            self.assertEqual('contents of foo\n', t.get('foo').read())
1214
 
        finally:
1215
 
            self._restore_env()
 
1294
    def assertProxied(self):
 
1295
        t = self.get_readonly_transport()
 
1296
        self.assertEqual('proxied contents of foo\n', t.get('foo').read())
 
1297
 
 
1298
    def assertNotProxied(self):
 
1299
        t = self.get_readonly_transport()
 
1300
        self.assertEqual('contents of foo\n', t.get('foo').read())
1216
1301
 
1217
1302
    def test_http_proxy(self):
1218
 
        self.proxied_in_env({'http_proxy': self.proxy_url})
 
1303
        self.overrideEnv('http_proxy', self.proxy_url)
 
1304
        self.assertProxied()
1219
1305
 
1220
1306
    def test_HTTP_PROXY(self):
1221
1307
        if self._testing_pycurl():
1224
1310
            # about. Should we ?)
1225
1311
            raise tests.TestNotApplicable(
1226
1312
                'pycurl does not check HTTP_PROXY for security reasons')
1227
 
        self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
 
1313
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1314
        self.assertProxied()
1228
1315
 
1229
1316
    def test_all_proxy(self):
1230
 
        self.proxied_in_env({'all_proxy': self.proxy_url})
 
1317
        self.overrideEnv('all_proxy', self.proxy_url)
 
1318
        self.assertProxied()
1231
1319
 
1232
1320
    def test_ALL_PROXY(self):
1233
 
        self.proxied_in_env({'ALL_PROXY': self.proxy_url})
 
1321
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1322
        self.assertProxied()
1234
1323
 
1235
1324
    def test_http_proxy_with_no_proxy(self):
1236
 
        self.not_proxied_in_env({'http_proxy': self.proxy_url,
1237
 
                                 'no_proxy': self.no_proxy_host})
 
1325
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1326
        self.overrideEnv('http_proxy', self.proxy_url)
 
1327
        self.assertNotProxied()
1238
1328
 
1239
1329
    def test_HTTP_PROXY_with_NO_PROXY(self):
1240
1330
        if self._testing_pycurl():
1241
1331
            raise tests.TestNotApplicable(
1242
1332
                'pycurl does not check HTTP_PROXY for security reasons')
1243
 
        self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
1244
 
                                 'NO_PROXY': self.no_proxy_host})
 
1333
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1334
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1335
        self.assertNotProxied()
1245
1336
 
1246
1337
    def test_all_proxy_with_no_proxy(self):
1247
 
        self.not_proxied_in_env({'all_proxy': self.proxy_url,
1248
 
                                 'no_proxy': self.no_proxy_host})
 
1338
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1339
        self.overrideEnv('all_proxy', self.proxy_url)
 
1340
        self.assertNotProxied()
1249
1341
 
1250
1342
    def test_ALL_PROXY_with_NO_PROXY(self):
1251
 
        self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
1252
 
                                 'NO_PROXY': self.no_proxy_host})
 
1343
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1344
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1345
        self.assertNotProxied()
1253
1346
 
1254
1347
    def test_http_proxy_without_scheme(self):
 
1348
        self.overrideEnv('http_proxy', self.server_host_port)
1255
1349
        if self._testing_pycurl():
1256
1350
            # pycurl *ignores* invalid proxy env variables. If that ever change
1257
1351
            # in the future, this test will fail indicating that pycurl do not
1258
1352
            # ignore anymore such variables.
1259
 
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
 
1353
            self.assertNotProxied()
1260
1354
        else:
1261
 
            self.assertRaises(errors.InvalidURL,
1262
 
                              self.proxied_in_env,
1263
 
                              {'http_proxy': self.proxy_address})
 
1355
            self.assertRaises(errors.InvalidURL, self.assertProxied)
1264
1356
 
1265
1357
 
1266
1358
class TestRanges(http_utils.TestCaseWithWebserver):
1267
1359
    """Test the Range header in GET methods."""
1268
1360
 
 
1361
    scenarios = multiply_scenarios(
 
1362
        vary_by_http_client_implementation(),
 
1363
        vary_by_http_protocol_version(),
 
1364
        )
 
1365
 
1269
1366
    def setUp(self):
1270
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1367
        super(TestRanges, self).setUp()
1271
1368
        self.build_tree_contents([('a', '0123456789')],)
1272
 
        server = self.get_readonly_server()
1273
 
        self.transport = self._transport(server.get_url())
1274
1369
 
1275
1370
    def create_transport_readonly_server(self):
1276
1371
        return http_server.HttpServer(protocol_version=self._protocol_version)
1277
1372
 
1278
1373
    def _file_contents(self, relpath, ranges):
 
1374
        t = self.get_readonly_transport()
1279
1375
        offsets = [ (start, end - start + 1) for start, end in ranges]
1280
 
        coalesce = self.transport._coalesce_offsets
 
1376
        coalesce = t._coalesce_offsets
1281
1377
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1282
 
        code, data = self.transport._get(relpath, coalesced)
 
1378
        code, data = t._get(relpath, coalesced)
1283
1379
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1284
1380
        for start, end in ranges:
1285
1381
            data.seek(start)
1286
1382
            yield data.read(end - start + 1)
1287
1383
 
1288
1384
    def _file_tail(self, relpath, tail_amount):
1289
 
        code, data = self.transport._get(relpath, [], tail_amount)
 
1385
        t = self.get_readonly_transport()
 
1386
        code, data = t._get(relpath, [], tail_amount)
1290
1387
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1291
1388
        data.seek(-tail_amount, 2)
1292
1389
        return data.read(tail_amount)
1311
1408
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1312
1409
    """Test redirection between http servers."""
1313
1410
 
1314
 
    def create_transport_secondary_server(self):
1315
 
        """Create the secondary server redirecting to the primary server"""
1316
 
        new = self.get_readonly_server()
1317
 
 
1318
 
        redirecting = http_utils.HTTPServerRedirecting(
1319
 
            protocol_version=self._protocol_version)
1320
 
        redirecting.redirect_to(new.host, new.port)
1321
 
        return redirecting
 
1411
    scenarios = multiply_scenarios(
 
1412
        vary_by_http_client_implementation(),
 
1413
        vary_by_http_protocol_version(),
 
1414
        )
1322
1415
 
1323
1416
    def setUp(self):
1324
1417
        super(TestHTTPRedirections, self).setUp()
1326
1419
                                  ('bundle',
1327
1420
                                  '# Bazaar revision bundle v0.9\n#\n')
1328
1421
                                  ],)
1329
 
        # The requests to the old server will be redirected to the new server
1330
 
        self.old_transport = self._transport(self.old_server.get_url())
1331
1422
 
1332
1423
    def test_redirected(self):
1333
 
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1334
 
        t = self._transport(self.new_server.get_url())
1335
 
        self.assertEqual('0123456789', t.get('a').read())
1336
 
 
1337
 
    def test_read_redirected_bundle_from_url(self):
1338
 
        from bzrlib.bundle import read_bundle_from_url
1339
 
        url = self.old_transport.abspath('bundle')
1340
 
        bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
1341
 
                read_bundle_from_url, url)
1342
 
        # If read_bundle_from_url was successful we get an empty bundle
1343
 
        self.assertEqual([], bundle.revisions)
 
1424
        self.assertRaises(errors.RedirectRequested,
 
1425
                          self.get_old_transport().get, 'a')
 
1426
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1344
1427
 
1345
1428
 
1346
1429
class RedirectedRequest(_urllib2_wrappers.Request):
1363
1446
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1364
1447
 
1365
1448
 
 
1449
def cleanup_http_redirection_connections(test):
 
1450
    # Some sockets are opened but never seen by _urllib, so we trap them at
 
1451
    # the _urllib2_wrappers level to be able to clean them up.
 
1452
    def socket_disconnect(sock):
 
1453
        try:
 
1454
            sock.shutdown(socket.SHUT_RDWR)
 
1455
            sock.close()
 
1456
        except socket.error:
 
1457
            pass
 
1458
    def connect(connection):
 
1459
        test.http_connect_orig(connection)
 
1460
        test.addCleanup(socket_disconnect, connection.sock)
 
1461
    test.http_connect_orig = test.overrideAttr(
 
1462
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
 
1463
    def connect(connection):
 
1464
        test.https_connect_orig(connection)
 
1465
        test.addCleanup(socket_disconnect, connection.sock)
 
1466
    test.https_connect_orig = test.overrideAttr(
 
1467
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
 
1468
 
 
1469
 
1366
1470
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1367
1471
    """Test redirections.
1368
1472
 
1377
1481
    -- vila 20070212
1378
1482
    """
1379
1483
 
 
1484
    scenarios = multiply_scenarios(
 
1485
        vary_by_http_client_implementation(),
 
1486
        vary_by_http_protocol_version(),
 
1487
        )
 
1488
 
1380
1489
    def setUp(self):
1381
1490
        if (features.pycurl.available()
1382
1491
            and self._transport == PyCurlTransport):
1383
1492
            raise tests.TestNotApplicable(
1384
 
                "pycurl doesn't redirect silently annymore")
 
1493
                "pycurl doesn't redirect silently anymore")
1385
1494
        super(TestHTTPSilentRedirections, self).setUp()
1386
1495
        install_redirected_request(self)
 
1496
        cleanup_http_redirection_connections(self)
1387
1497
        self.build_tree_contents([('a','a'),
1388
1498
                                  ('1/',),
1389
1499
                                  ('1/a', 'redirected once'),
1397
1507
                                  ('5/a', 'redirected 5 times'),
1398
1508
                                  ],)
1399
1509
 
1400
 
        self.old_transport = self._transport(self.old_server.get_url())
1401
 
 
1402
 
    def create_transport_secondary_server(self):
1403
 
        """Create the secondary server, redirections are defined in the tests"""
1404
 
        return http_utils.HTTPServerRedirecting(
1405
 
            protocol_version=self._protocol_version)
1406
 
 
1407
1510
    def test_one_redirection(self):
1408
 
        t = self.old_transport
1409
 
 
1410
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1511
        t = self.get_old_transport()
 
1512
        req = RedirectedRequest('GET', t._remote_path('a'))
1411
1513
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1412
1514
                                       self.new_server.port)
1413
1515
        self.old_server.redirections = \
1414
1516
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1415
 
        self.assertEqual('redirected once',t._perform(req).read())
 
1517
        self.assertEqual('redirected once', t._perform(req).read())
1416
1518
 
1417
1519
    def test_five_redirections(self):
1418
 
        t = self.old_transport
1419
 
 
1420
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1520
        t = self.get_old_transport()
 
1521
        req = RedirectedRequest('GET', t._remote_path('a'))
1421
1522
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1422
1523
                                       self.old_server.port)
1423
1524
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1429
1530
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1430
1531
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1431
1532
            ]
1432
 
        self.assertEqual('redirected 5 times',t._perform(req).read())
 
1533
        self.assertEqual('redirected 5 times', t._perform(req).read())
1433
1534
 
1434
1535
 
1435
1536
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1436
1537
    """Test transport.do_catching_redirections."""
1437
1538
 
 
1539
    scenarios = multiply_scenarios(
 
1540
        vary_by_http_client_implementation(),
 
1541
        vary_by_http_protocol_version(),
 
1542
        )
 
1543
 
1438
1544
    def setUp(self):
1439
1545
        super(TestDoCatchRedirections, self).setUp()
1440
1546
        self.build_tree_contents([('a', '0123456789'),],)
1441
 
 
1442
 
        self.old_transport = self._transport(self.old_server.get_url())
1443
 
 
1444
 
    def get_a(self, transport):
1445
 
        return transport.get('a')
 
1547
        cleanup_http_redirection_connections(self)
 
1548
 
 
1549
        self.old_transport = self.get_old_transport()
 
1550
 
 
1551
    def get_a(self, t):
 
1552
        return t.get('a')
1446
1553
 
1447
1554
    def test_no_redirection(self):
1448
 
        t = self._transport(self.new_server.get_url())
 
1555
        t = self.get_new_transport()
1449
1556
 
1450
1557
        # We use None for redirected so that we fail if redirected
1451
1558
        self.assertEqual('0123456789',
1455
1562
    def test_one_redirection(self):
1456
1563
        self.redirections = 0
1457
1564
 
1458
 
        def redirected(transport, exception, redirection_notice):
 
1565
        def redirected(t, exception, redirection_notice):
1459
1566
            self.redirections += 1
1460
 
            dir, file = urlutils.split(exception.target)
1461
 
            return self._transport(dir)
 
1567
            redirected_t = t._redirected_to(exception.source, exception.target)
 
1568
            return redirected_t
1462
1569
 
1463
1570
        self.assertEqual('0123456789',
1464
1571
                         transport.do_catching_redirections(
1478
1585
                          self.get_a, self.old_transport, redirected)
1479
1586
 
1480
1587
 
 
1588
def _setup_authentication_config(**kwargs):
 
1589
    conf = config.AuthenticationConfig()
 
1590
    conf._get_config().update({'httptest': kwargs})
 
1591
    conf._save()
 
1592
 
 
1593
 
 
1594
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
 
1595
    """Unit tests for glue by which urllib2 asks us for authentication"""
 
1596
 
 
1597
    def test_get_user_password_without_port(self):
 
1598
        """We cope if urllib2 doesn't tell us the port.
 
1599
 
 
1600
        See https://bugs.launchpad.net/bzr/+bug/654684
 
1601
        """
 
1602
        user = 'joe'
 
1603
        password = 'foo'
 
1604
        _setup_authentication_config(scheme='http', host='localhost',
 
1605
                                     user=user, password=password)
 
1606
        handler = _urllib2_wrappers.HTTPAuthHandler()
 
1607
        got_pass = handler.get_user_password(dict(
 
1608
            user='joe',
 
1609
            protocol='http',
 
1610
            host='localhost',
 
1611
            path='/',
 
1612
            realm='Realm',
 
1613
            ))
 
1614
        self.assertEqual((user, password), got_pass)
 
1615
 
 
1616
 
1481
1617
class TestAuth(http_utils.TestCaseWithWebserver):
1482
1618
    """Test authentication scheme"""
1483
1619
 
1484
 
    _auth_header = 'Authorization'
1485
 
    _password_prompt_prefix = ''
1486
 
    _username_prompt_prefix = ''
1487
 
    # Set by load_tests
1488
 
    _auth_server = None
 
1620
    scenarios = multiply_scenarios(
 
1621
        vary_by_http_client_implementation(),
 
1622
        vary_by_http_protocol_version(),
 
1623
        vary_by_http_auth_scheme(),
 
1624
        )
1489
1625
 
1490
1626
    def setUp(self):
1491
1627
        super(TestAuth, self).setUp()
1494
1630
                                  ('b', 'contents of b\n'),])
1495
1631
 
1496
1632
    def create_transport_readonly_server(self):
1497
 
        return self._auth_server(protocol_version=self._protocol_version)
 
1633
        server = self._auth_server(protocol_version=self._protocol_version)
 
1634
        server._url_protocol = self._url_protocol
 
1635
        return server
1498
1636
 
1499
1637
    def _testing_pycurl(self):
1500
1638
        # TODO: This is duplicated for lots of the classes in this file
1513
1651
        return url
1514
1652
 
1515
1653
    def get_user_transport(self, user, password):
1516
 
        return self._transport(self.get_user_url(user, password))
 
1654
        t = transport.get_transport_from_url(
 
1655
            self.get_user_url(user, password))
 
1656
        return t
1517
1657
 
1518
1658
    def test_no_user(self):
1519
1659
        self.server.add_user('joe', 'foo')
1630
1770
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1631
1771
                                            stderr=tests.StringIOWrapper())
1632
1772
        # Create a minimal config file with the right password
1633
 
        conf = config.AuthenticationConfig()
1634
 
        conf._get_config().update(
1635
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1636
 
                          'user': user, 'password': password}})
1637
 
        conf._save()
 
1773
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1774
                                     user=user, password=password)
1638
1775
        # Issue a request to the server to connect
1639
1776
        self.assertEqual('contents of a\n',t.get('a').read())
1640
1777
        # stdin should have  been left untouched
1642
1779
        # Only one 'Authentication Required' error should occur
1643
1780
        self.assertEqual(1, self.server.auth_required_errors)
1644
1781
 
1645
 
    def test_user_from_auth_conf(self):
1646
 
        if self._testing_pycurl():
1647
 
            raise tests.TestNotApplicable(
1648
 
                'pycurl does not support authentication.conf')
1649
 
        user = 'joe'
1650
 
        password = 'foo'
1651
 
        self.server.add_user(user, password)
1652
 
        # Create a minimal config file with the right password
1653
 
        conf = config.AuthenticationConfig()
1654
 
        conf._get_config().update(
1655
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1656
 
                          'user': user, 'password': password}})
1657
 
        conf._save()
1658
 
        t = self.get_user_transport(None, None)
1659
 
        # Issue a request to the server to connect
1660
 
        self.assertEqual('contents of a\n', t.get('a').read())
1661
 
        # Only one 'Authentication Required' error should occur
1662
 
        self.assertEqual(1, self.server.auth_required_errors)
1663
 
 
1664
1782
    def test_changing_nonce(self):
1665
1783
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1666
1784
                                     http_utils.ProxyDigestAuthServer):
1667
1785
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1668
1786
        if self._testing_pycurl():
1669
 
            raise tests.KnownFailure(
 
1787
            self.knownFailure(
1670
1788
                'pycurl does not handle a nonce change')
1671
1789
        self.server.add_user('joe', 'foo')
1672
1790
        t = self.get_user_transport('joe', 'foo')
1682
1800
        # initial 'who are you' and a second 'who are you' with the new nonce)
1683
1801
        self.assertEqual(2, self.server.auth_required_errors)
1684
1802
 
 
1803
    def test_user_from_auth_conf(self):
 
1804
        if self._testing_pycurl():
 
1805
            raise tests.TestNotApplicable(
 
1806
                'pycurl does not support authentication.conf')
 
1807
        user = 'joe'
 
1808
        password = 'foo'
 
1809
        self.server.add_user(user, password)
 
1810
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1811
                                     user=user, password=password)
 
1812
        t = self.get_user_transport(None, None)
 
1813
        # Issue a request to the server to connect
 
1814
        self.assertEqual('contents of a\n', t.get('a').read())
 
1815
        # Only one 'Authentication Required' error should occur
 
1816
        self.assertEqual(1, self.server.auth_required_errors)
 
1817
 
 
1818
    def test_no_credential_leaks_in_log(self):
 
1819
        self.overrideAttr(debug, 'debug_flags', set(['http']))
 
1820
        user = 'joe'
 
1821
        password = 'very-sensitive-password'
 
1822
        self.server.add_user(user, password)
 
1823
        t = self.get_user_transport(user, password)
 
1824
        # Capture the debug calls to mutter
 
1825
        self.mutters = []
 
1826
        def mutter(*args):
 
1827
            lines = args[0] % args[1:]
 
1828
            # Some calls output multiple lines, just split them now since we
 
1829
            # care about a single one later.
 
1830
            self.mutters.extend(lines.splitlines())
 
1831
        self.overrideAttr(trace, 'mutter', mutter)
 
1832
        # Issue a request to the server to connect
 
1833
        self.assertEqual(True, t.has('a'))
 
1834
        # Only one 'Authentication Required' error should occur
 
1835
        self.assertEqual(1, self.server.auth_required_errors)
 
1836
        # Since the authentification succeeded, there should be a corresponding
 
1837
        # debug line
 
1838
        sent_auth_headers = [line for line in self.mutters
 
1839
                             if line.startswith('> %s' % (self._auth_header,))]
 
1840
        self.assertLength(1, sent_auth_headers)
 
1841
        self.assertStartsWith(sent_auth_headers[0],
 
1842
                              '> %s: <masked>' % (self._auth_header,))
1685
1843
 
1686
1844
 
1687
1845
class TestProxyAuth(TestAuth):
1688
 
    """Test proxy authentication schemes."""
1689
 
 
1690
 
    _auth_header = 'Proxy-authorization'
1691
 
    _password_prompt_prefix = 'Proxy '
1692
 
    _username_prompt_prefix = 'Proxy '
 
1846
    """Test proxy authentication schemes.
 
1847
 
 
1848
    This inherits from TestAuth to tweak the setUp and filter some failing
 
1849
    tests.
 
1850
    """
 
1851
 
 
1852
    scenarios = multiply_scenarios(
 
1853
        vary_by_http_client_implementation(),
 
1854
        vary_by_http_protocol_version(),
 
1855
        vary_by_http_proxy_auth_scheme(),
 
1856
        )
1693
1857
 
1694
1858
    def setUp(self):
1695
1859
        super(TestProxyAuth, self).setUp()
1696
 
        self._old_env = {}
1697
 
        self.addCleanup(self._restore_env)
1698
1860
        # Override the contents to avoid false positives
1699
1861
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1700
1862
                                  ('b', 'not proxied contents of b\n'),
1703
1865
                                  ])
1704
1866
 
1705
1867
    def get_user_transport(self, user, password):
1706
 
        self._install_env({'all_proxy': self.get_user_url(user, password)})
1707
 
        return self._transport(self.server.get_url())
1708
 
 
1709
 
    def _install_env(self, env):
1710
 
        for name, value in env.iteritems():
1711
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1712
 
 
1713
 
    def _restore_env(self):
1714
 
        for name, value in self._old_env.iteritems():
1715
 
            osutils.set_or_unset_env(name, value)
 
1868
        proxy_url = self.get_user_url(user, password)
 
1869
        if self._testing_pycurl():
 
1870
            proxy_url = proxy_url.replace('+pycurl', '')
 
1871
        self.overrideEnv('all_proxy', proxy_url)
 
1872
        return TestAuth.get_user_transport(self, user, password)
1716
1873
 
1717
1874
    def test_empty_pass(self):
1718
1875
        if self._testing_pycurl():
1719
1876
            import pycurl
1720
1877
            if pycurl.version_info()[1] < '7.16.0':
1721
 
                raise tests.KnownFailure(
 
1878
                self.knownFailure(
1722
1879
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1723
1880
        super(TestProxyAuth, self).test_empty_pass()
1724
1881
 
1737
1894
        self.readfile = StringIO(socket_read_content)
1738
1895
        self.writefile = StringIO()
1739
1896
        self.writefile.close = lambda: None
 
1897
        self.close = lambda: None
1740
1898
 
1741
1899
    def makefile(self, mode='r', bufsize=None):
1742
1900
        if 'r' in mode:
1747
1905
 
1748
1906
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1749
1907
 
 
1908
    scenarios = multiply_scenarios(
 
1909
        vary_by_http_client_implementation(),
 
1910
        vary_by_http_protocol_version(),
 
1911
        )
 
1912
 
1750
1913
    def setUp(self):
1751
1914
        super(SmartHTTPTunnellingTest, self).setUp()
1752
1915
        # We use the VFS layer as part of HTTP tunnelling tests.
1753
 
        self._captureVar('BZR_NO_SMART_VFS', None)
 
1916
        self.overrideEnv('BZR_NO_SMART_VFS', None)
1754
1917
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
 
1918
        self.http_server = self.get_readonly_server()
1755
1919
 
1756
1920
    def create_transport_readonly_server(self):
1757
 
        return http_utils.HTTPServerWithSmarts(
 
1921
        server = http_utils.HTTPServerWithSmarts(
1758
1922
            protocol_version=self._protocol_version)
 
1923
        server._url_protocol = self._url_protocol
 
1924
        return server
1759
1925
 
1760
 
    def test_open_bzrdir(self):
 
1926
    def test_open_controldir(self):
1761
1927
        branch = self.make_branch('relpath')
1762
 
        http_server = self.get_readonly_server()
1763
 
        url = http_server.get_url() + 'relpath'
1764
 
        bd = bzrdir.BzrDir.open(url)
 
1928
        url = self.http_server.get_url() + 'relpath'
 
1929
        bd = controldir.ControlDir.open(url)
 
1930
        self.addCleanup(bd.transport.disconnect)
1765
1931
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1766
1932
 
1767
1933
    def test_bulk_data(self):
1769
1935
        # The 'readv' command in the smart protocol both sends and receives
1770
1936
        # bulk data, so we use that.
1771
1937
        self.build_tree(['data-file'])
1772
 
        http_server = self.get_readonly_server()
1773
 
        http_transport = self._transport(http_server.get_url())
 
1938
        http_transport = transport.get_transport_from_url(
 
1939
            self.http_server.get_url())
1774
1940
        medium = http_transport.get_smart_medium()
1775
1941
        # Since we provide the medium, the url below will be mostly ignored
1776
1942
        # during the test, as long as the path is '/'.
1784
1950
        post_body = 'hello\n'
1785
1951
        expected_reply_body = 'ok\x012\n'
1786
1952
 
1787
 
        http_server = self.get_readonly_server()
1788
 
        http_transport = self._transport(http_server.get_url())
 
1953
        http_transport = transport.get_transport_from_url(
 
1954
            self.http_server.get_url())
1789
1955
        medium = http_transport.get_smart_medium()
1790
1956
        response = medium.send_http_smart_request(post_body)
1791
1957
        reply_body = response.read()
1792
1958
        self.assertEqual(expected_reply_body, reply_body)
1793
1959
 
1794
1960
    def test_smart_http_server_post_request_handler(self):
1795
 
        httpd = self.get_readonly_server()._get_httpd()
 
1961
        httpd = self.http_server.server
1796
1962
 
1797
1963
        socket = SampleSocket(
1798
1964
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1830
1996
 
1831
1997
    def test_probe_smart_server(self):
1832
1998
        """Test error handling against server refusing smart requests."""
1833
 
        server = self.get_readonly_server()
1834
 
        t = self._transport(server.get_url())
 
1999
        t = self.get_readonly_transport()
1835
2000
        # No need to build a valid smart request here, the server will not even
1836
2001
        # try to interpret it.
1837
2002
        self.assertRaises(errors.SmartProtocolError,
1838
2003
                          t.get_smart_medium().send_http_smart_request,
1839
2004
                          'whatever')
1840
2005
 
 
2006
 
1841
2007
class Test_redirected_to(tests.TestCase):
1842
2008
 
 
2009
    scenarios = vary_by_http_client_implementation()
 
2010
 
1843
2011
    def test_redirected_to_subdir(self):
1844
2012
        t = self._transport('http://www.example.com/foo')
1845
2013
        r = t._redirected_to('http://www.example.com/foo',
1847
2015
        self.assertIsInstance(r, type(t))
1848
2016
        # Both transports share the some connection
1849
2017
        self.assertEqual(t._get_connection(), r._get_connection())
 
2018
        self.assertEqual('http://www.example.com/foo/subdir/', r.base)
1850
2019
 
1851
2020
    def test_redirected_to_self_with_slash(self):
1852
2021
        t = self._transport('http://www.example.com/foo')
1863
2032
        r = t._redirected_to('http://www.example.com/foo',
1864
2033
                             'http://foo.example.com/foo/subdir')
1865
2034
        self.assertIsInstance(r, type(t))
 
2035
        self.assertEqual('http://foo.example.com/foo/subdir/',
 
2036
            r.external_url())
1866
2037
 
1867
2038
    def test_redirected_to_same_host_sibling_protocol(self):
1868
2039
        t = self._transport('http://www.example.com/foo')
1869
2040
        r = t._redirected_to('http://www.example.com/foo',
1870
2041
                             'https://www.example.com/foo')
1871
2042
        self.assertIsInstance(r, type(t))
 
2043
        self.assertEqual('https://www.example.com/foo/',
 
2044
            r.external_url())
1872
2045
 
1873
2046
    def test_redirected_to_same_host_different_protocol(self):
1874
2047
        t = self._transport('http://www.example.com/foo')
1875
2048
        r = t._redirected_to('http://www.example.com/foo',
1876
2049
                             'ftp://www.example.com/foo')
1877
 
        self.assertNotEquals(type(r), type(t))
 
2050
        self.assertNotEqual(type(r), type(t))
 
2051
        self.assertEqual('ftp://www.example.com/foo/', r.external_url())
 
2052
 
 
2053
    def test_redirected_to_same_host_specific_implementation(self):
 
2054
        t = self._transport('http://www.example.com/foo')
 
2055
        r = t._redirected_to('http://www.example.com/foo',
 
2056
                             'https+urllib://www.example.com/foo')
 
2057
        self.assertEqual('https://www.example.com/foo/', r.external_url())
1878
2058
 
1879
2059
    def test_redirected_to_different_host_same_user(self):
1880
2060
        t = self._transport('http://joe@www.example.com/foo')
1881
2061
        r = t._redirected_to('http://www.example.com/foo',
1882
2062
                             'https://foo.example.com/foo')
1883
2063
        self.assertIsInstance(r, type(t))
1884
 
        self.assertEqual(t._user, r._user)
 
2064
        self.assertEqual(t._parsed_url.user, r._parsed_url.user)
 
2065
        self.assertEqual('https://joe@foo.example.com/foo/', r.external_url())
1885
2066
 
1886
2067
 
1887
2068
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1896
2077
    line.
1897
2078
    """
1898
2079
 
1899
 
    def handle_one_request(self):
 
2080
    def _handle_one_request(self):
1900
2081
        tcs = self.server.test_case_server
1901
2082
        requestline = self.rfile.readline()
1902
2083
        headers = self.MessageClass(self.rfile, 0)
1940
2121
    pass
1941
2122
 
1942
2123
 
1943
 
if tests.HTTPSServerFeature.available():
1944
 
    from bzrlib.tests import https_server
 
2124
if features.HTTPSServerFeature.available():
 
2125
    from brzlib.tests import https_server
1945
2126
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
1946
2127
        pass
1947
2128
 
1954
2135
    """
1955
2136
 
1956
2137
    def setUp(self):
1957
 
        tests.TestCase.setUp(self)
1958
2138
        self.server = self._activity_server(self._protocol_version)
1959
2139
        self.server.start_server()
1960
 
        self.activities = {}
 
2140
        self.addCleanup(self.server.stop_server)
 
2141
        _activities = {} # Don't close over self and create a cycle
1961
2142
        def report_activity(t, bytes, direction):
1962
 
            count = self.activities.get(direction, 0)
 
2143
            count = _activities.get(direction, 0)
1963
2144
            count += bytes
1964
 
            self.activities[direction] = count
1965
 
 
 
2145
            _activities[direction] = count
 
2146
        self.activities = _activities
1966
2147
        # We override at class level because constructors may propagate the
1967
2148
        # bound method and render instance overriding ineffective (an
1968
2149
        # alternative would be to define a specific ui factory instead...)
1969
 
        self.orig_report_activity = self._transport._report_activity
1970
 
        self._transport._report_activity = report_activity
1971
 
 
1972
 
    def tearDown(self):
1973
 
        self._transport._report_activity = self.orig_report_activity
1974
 
        self.server.stop_server()
1975
 
        tests.TestCase.tearDown(self)
 
2150
        self.overrideAttr(self._transport, '_report_activity', report_activity)
1976
2151
 
1977
2152
    def get_transport(self):
1978
 
        return self._transport(self.server.get_url())
 
2153
        t = self._transport(self.server.get_url())
 
2154
        # FIXME: Needs cleanup -- vila 20100611
 
2155
        return t
1979
2156
 
1980
2157
    def assertActivitiesMatch(self):
1981
2158
        self.assertEqual(self.server.bytes_read,
2086
2263
'''
2087
2264
        t = self.get_transport()
2088
2265
        # We must send a single line of body bytes, see
2089
 
        # PredefinedRequestHandler.handle_one_request
 
2266
        # PredefinedRequestHandler._handle_one_request
2090
2267
        code, f = t._post('abc def end-of-body\n')
2091
2268
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2092
2269
        self.assertActivitiesMatch()
2094
2271
 
2095
2272
class TestActivity(tests.TestCase, TestActivityMixin):
2096
2273
 
 
2274
    scenarios = multiply_scenarios(
 
2275
        vary_by_http_activity(),
 
2276
        vary_by_http_protocol_version(),
 
2277
        )
 
2278
 
2097
2279
    def setUp(self):
2098
 
        tests.TestCase.setUp(self)
2099
 
        self.server = self._activity_server(self._protocol_version)
2100
 
        self.server.start_server()
2101
 
        self.activities = {}
2102
 
        def report_activity(t, bytes, direction):
2103
 
            count = self.activities.get(direction, 0)
2104
 
            count += bytes
2105
 
            self.activities[direction] = count
2106
 
 
2107
 
        # We override at class level because constructors may propagate the
2108
 
        # bound method and render instance overriding ineffective (an
2109
 
        # alternative would be to define a specific ui factory instead...)
2110
 
        self.orig_report_activity = self._transport._report_activity
2111
 
        self._transport._report_activity = report_activity
2112
 
 
2113
 
    def tearDown(self):
2114
 
        self._transport._report_activity = self.orig_report_activity
2115
 
        self.server.stop_server()
2116
 
        tests.TestCase.tearDown(self)
 
2280
        super(TestActivity, self).setUp()
 
2281
        TestActivityMixin.setUp(self)
2117
2282
 
2118
2283
 
2119
2284
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2120
2285
 
 
2286
    # Unlike TestActivity, we are really testing ReportingFileSocket and
 
2287
    # ReportingSocket, so we don't need all the parametrization. Since
 
2288
    # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
 
2289
    # test them through their use by the transport than directly (that's a
 
2290
    # bit less clean but far more simpler and effective).
 
2291
    _activity_server = ActivityHTTPServer
 
2292
    _protocol_version = 'HTTP/1.1'
 
2293
 
2121
2294
    def setUp(self):
2122
 
        tests.TestCase.setUp(self)
2123
 
        # Unlike TestActivity, we are really testing ReportingFileSocket and
2124
 
        # ReportingSocket, so we don't need all the parametrization. Since
2125
 
        # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2126
 
        # test them through their use by the transport than directly (that's a
2127
 
        # bit less clean but far more simpler and effective).
2128
 
        self.server = ActivityHTTPServer('HTTP/1.1')
2129
 
        self._transport=_urllib.HttpTransport_urllib
2130
 
 
2131
 
        self.server.start_server()
2132
 
 
2133
 
        # We override at class level because constructors may propagate the
2134
 
        # bound method and render instance overriding ineffective (an
2135
 
        # alternative would be to define a specific ui factory instead...)
2136
 
        self.orig_report_activity = self._transport._report_activity
2137
 
        self._transport._report_activity = None
2138
 
 
2139
 
    def tearDown(self):
2140
 
        self._transport._report_activity = self.orig_report_activity
2141
 
        self.server.stop_server()
2142
 
        tests.TestCase.tearDown(self)
 
2295
        super(TestNoReportActivity, self).setUp()
 
2296
        self._transport =_urllib.HttpTransport_urllib
 
2297
        TestActivityMixin.setUp(self)
2143
2298
 
2144
2299
    def assertActivitiesMatch(self):
2145
2300
        # Nothing to check here
2149
2304
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2150
2305
    """Test authentication on the redirected http server."""
2151
2306
 
 
2307
    scenarios = vary_by_http_protocol_version()
 
2308
 
2152
2309
    _auth_header = 'Authorization'
2153
2310
    _password_prompt_prefix = ''
2154
2311
    _username_prompt_prefix = ''
2155
2312
    _auth_server = http_utils.HTTPBasicAuthServer
2156
2313
    _transport = _urllib.HttpTransport_urllib
2157
2314
 
2158
 
    def create_transport_readonly_server(self):
2159
 
        return self._auth_server()
2160
 
 
2161
 
    def create_transport_secondary_server(self):
2162
 
        """Create the secondary server redirecting to the primary server"""
2163
 
        new = self.get_readonly_server()
2164
 
 
2165
 
        redirecting = http_utils.HTTPServerRedirecting()
2166
 
        redirecting.redirect_to(new.host, new.port)
2167
 
        return redirecting
2168
 
 
2169
2315
    def setUp(self):
2170
2316
        super(TestAuthOnRedirected, self).setUp()
2171
2317
        self.build_tree_contents([('a','a'),
2176
2322
                                       self.new_server.port)
2177
2323
        self.old_server.redirections = [
2178
2324
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2179
 
        self.old_transport = self._transport(self.old_server.get_url())
 
2325
        self.old_transport = self.get_old_transport()
2180
2326
        self.new_server.add_user('joe', 'foo')
2181
 
 
2182
 
    def get_a(self, transport):
2183
 
        return transport.get('a')
 
2327
        cleanup_http_redirection_connections(self)
 
2328
 
 
2329
    def create_transport_readonly_server(self):
 
2330
        server = self._auth_server(protocol_version=self._protocol_version)
 
2331
        server._url_protocol = self._url_protocol
 
2332
        return server
 
2333
 
 
2334
    def get_a(self, t):
 
2335
        return t.get('a')
2184
2336
 
2185
2337
    def test_auth_on_redirected_via_do_catching_redirections(self):
2186
2338
        self.redirections = 0
2187
2339
 
2188
 
        def redirected(transport, exception, redirection_notice):
 
2340
        def redirected(t, exception, redirection_notice):
2189
2341
            self.redirections += 1
2190
 
            dir, file = urlutils.split(exception.target)
2191
 
            return self._transport(dir)
 
2342
            redirected_t = t._redirected_to(exception.source, exception.target)
 
2343
            self.addCleanup(redirected_t.disconnect)
 
2344
            return redirected_t
2192
2345
 
2193
2346
        stdout = tests.StringIOWrapper()
2194
2347
        stderr = tests.StringIOWrapper()
2215
2368
                                       self.new_server.port)
2216
2369
        self.old_server.redirections = [
2217
2370
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2218
 
        self.assertEqual('redirected once',t._perform(req).read())
 
2371
        self.assertEqual('redirected once', t._perform(req).read())
2219
2372
        # stdin should be empty
2220
2373
        self.assertEqual('', ui.ui_factory.stdin.readline())
2221
2374
        # stdout should be empty, stderr will contains the prompts
2222
2375
        self.assertEqual('', stdout.getvalue())
2223
2376
 
2224