/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_http.py

(jameinel) Transform.rename test shouldn't assume the str form of the
 exception because of locale issues. (Martin [gz])

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
# Copyright (C) 2005-2010 Canonical Ltd
 
1
# Copyright (C) 2005-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
23
23
# TODO: Should be renamed to bzrlib.transport.http.tests?
24
24
# TODO: What about renaming to bzrlib.tests.transport.http ?
25
25
 
26
 
from cStringIO import StringIO
27
26
import httplib
28
 
import os
29
 
import select
30
27
import SimpleHTTPServer
31
28
import socket
32
29
import sys
42
39
    tests,
43
40
    transport,
44
41
    ui,
45
 
    urlutils,
46
 
    )
47
 
from bzrlib.symbol_versioning import (
48
 
    deprecated_in,
49
42
    )
50
43
from bzrlib.tests import (
51
44
    features,
52
45
    http_server,
53
46
    http_utils,
 
47
    test_server,
 
48
    )
 
49
from bzrlib.tests.scenarios import (
 
50
    load_tests_apply_scenarios,
 
51
    multiply_scenarios,
54
52
    )
55
53
from bzrlib.transport import (
56
54
    http,
66
64
    from bzrlib.transport.http._pycurl import PyCurlTransport
67
65
 
68
66
 
69
 
def load_tests(standard_tests, module, loader):
70
 
    """Multiply tests for http clients and protocol versions."""
71
 
    result = loader.suiteClass()
72
 
 
73
 
    # one for each transport implementation
74
 
    t_tests, remaining_tests = tests.split_suite_by_condition(
75
 
        standard_tests, tests.condition_isinstance((
76
 
                TestHttpTransportRegistration,
77
 
                TestHttpTransportUrls,
78
 
                Test_redirected_to,
79
 
                )))
 
67
load_tests = load_tests_apply_scenarios
 
68
 
 
69
 
 
70
def vary_by_http_client_implementation():
 
71
    """Test the two libraries we can use, pycurl and urllib."""
80
72
    transport_scenarios = [
81
73
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
82
74
                        _server=http_server.HttpServer_urllib,
83
 
                        _qualified_prefix='http+urllib',)),
 
75
                        _url_protocol='http+urllib',)),
84
76
        ]
85
77
    if features.pycurl.available():
86
78
        transport_scenarios.append(
87
79
            ('pycurl', dict(_transport=PyCurlTransport,
88
80
                            _server=http_server.HttpServer_PyCurl,
89
 
                            _qualified_prefix='http+pycurl',)))
90
 
    tests.multiply_tests(t_tests, transport_scenarios, result)
91
 
 
92
 
    # each implementation tested with each HTTP version
93
 
    tp_tests, remaining_tests = tests.split_suite_by_condition(
94
 
        remaining_tests, tests.condition_isinstance((
95
 
                SmartHTTPTunnellingTest,
96
 
                TestDoCatchRedirections,
97
 
                TestHTTPConnections,
98
 
                TestHTTPRedirections,
99
 
                TestHTTPSilentRedirections,
100
 
                TestLimitedRangeRequestServer,
101
 
                TestPost,
102
 
                TestProxyHttpServer,
103
 
                TestRanges,
104
 
                TestSpecificRequestHandler,
105
 
                )))
106
 
    protocol_scenarios = [
107
 
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
108
 
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
109
 
            ]
110
 
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
111
 
                                            protocol_scenarios)
112
 
    tests.multiply_tests(tp_tests, tp_scenarios, result)
113
 
 
114
 
    # proxy auth: each auth scheme on all http versions on all implementations.
115
 
    tppa_tests, remaining_tests = tests.split_suite_by_condition(
116
 
        remaining_tests, tests.condition_isinstance((
117
 
                TestProxyAuth,
118
 
                )))
119
 
    proxy_auth_scheme_scenarios = [
 
81
                            _url_protocol='http+pycurl',)))
 
82
    return transport_scenarios
 
83
 
 
84
 
 
85
def vary_by_http_protocol_version():
 
86
    """Test on http/1.0 and 1.1"""
 
87
    return [
 
88
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
89
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
 
90
        ]
 
91
 
 
92
 
 
93
def vary_by_http_proxy_auth_scheme():
 
94
    return [
120
95
        ('basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
121
96
        ('digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
122
97
        ('basicdigest',
123
 
         dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
98
            dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
124
99
        ]
125
 
    tppa_scenarios = tests.multiply_scenarios(tp_scenarios,
126
 
                                              proxy_auth_scheme_scenarios)
127
 
    tests.multiply_tests(tppa_tests, tppa_scenarios, result)
128
 
 
129
 
    # auth: each auth scheme on all http versions on all implementations.
130
 
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
131
 
        remaining_tests, tests.condition_isinstance((
132
 
                TestAuth,
133
 
                )))
134
 
    auth_scheme_scenarios = [
 
100
 
 
101
 
 
102
def vary_by_http_auth_scheme():
 
103
    return [
135
104
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
136
105
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
137
106
        ('basicdigest',
138
 
         dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
 
107
            dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
139
108
        ]
140
 
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
141
 
                                             auth_scheme_scenarios)
142
 
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
143
 
 
144
 
    # activity: on all http[s] versions on all implementations
145
 
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
146
 
        remaining_tests, tests.condition_isinstance((
147
 
                TestActivity,
148
 
                )))
 
109
 
 
110
 
 
111
def vary_by_http_activity():
149
112
    activity_scenarios = [
150
113
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
151
 
                             _transport=_urllib.HttpTransport_urllib,)),
 
114
                            _transport=_urllib.HttpTransport_urllib,)),
152
115
        ]
153
116
    if tests.HTTPSServerFeature.available():
154
117
        activity_scenarios.append(
155
118
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
156
 
                                  _transport=_urllib.HttpTransport_urllib,)),)
 
119
                                _transport=_urllib.HttpTransport_urllib,)),)
157
120
    if features.pycurl.available():
158
121
        activity_scenarios.append(
159
122
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
160
 
                                 _transport=PyCurlTransport,)),)
 
123
                                _transport=PyCurlTransport,)),)
161
124
        if tests.HTTPSServerFeature.available():
162
125
            from bzrlib.tests import (
163
126
                ssl_certs,
175
138
 
176
139
            activity_scenarios.append(
177
140
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
178
 
                                      _transport=HTTPS_pycurl_transport,)),)
179
 
 
180
 
    tpact_scenarios = tests.multiply_scenarios(activity_scenarios,
181
 
                                               protocol_scenarios)
182
 
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
183
 
 
184
 
    # No parametrization for the remaining tests
185
 
    result.addTests(remaining_tests)
186
 
 
187
 
    return result
 
141
                                    _transport=HTTPS_pycurl_transport,)),)
 
142
    return activity_scenarios
188
143
 
189
144
 
190
145
class FakeManager(object):
223
178
        self._sock.bind(('127.0.0.1', 0))
224
179
        self.host, self.port = self._sock.getsockname()
225
180
        self._ready = threading.Event()
226
 
        self._thread = threading.Thread(target=self._accept_read_and_reply)
227
 
        self._thread.setDaemon(True)
 
181
        self._thread = test_server.ThreadWithException(
 
182
            event=self._ready, target=self._accept_read_and_reply)
228
183
        self._thread.start()
229
 
        self._ready.wait(5)
 
184
        if 'threads' in tests.selftest_debug_flags:
 
185
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
 
186
        self._ready.wait()
230
187
 
231
188
    def _accept_read_and_reply(self):
232
189
        self._sock.listen(1)
233
190
        self._ready.set()
234
 
        self._sock.settimeout(5)
235
 
        try:
236
 
            conn, address = self._sock.accept()
237
 
            # On win32, the accepted connection will be non-blocking to start
238
 
            # with because we're using settimeout.
239
 
            conn.setblocking(True)
 
191
        conn, address = self._sock.accept()
 
192
        if self._expect_body_tail is not None:
240
193
            while not self.received_bytes.endswith(self._expect_body_tail):
241
194
                self.received_bytes += conn.recv(4096)
242
195
            conn.sendall('HTTP/1.1 200 OK\r\n')
243
 
        except socket.timeout:
244
 
            # Make sure the client isn't stuck waiting for us to e.g. accept.
 
196
        try:
245
197
            self._sock.close()
246
198
        except socket.error:
247
199
            # The client may have already closed the socket.
249
201
 
250
202
    def stop_server(self):
251
203
        try:
252
 
            self._sock.close()
 
204
            # Issue a fake connection to wake up the server and allow it to
 
205
            # finish quickly
 
206
            fake_conn = osutils.connect_socket((self.host, self.port))
 
207
            fake_conn.close()
253
208
        except socket.error:
254
209
            # We might have already closed it.  We don't care.
255
210
            pass
256
211
        self.host = None
257
212
        self.port = None
 
213
        self._thread.join()
 
214
        if 'threads' in tests.selftest_debug_flags:
 
215
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
258
216
 
259
217
 
260
218
class TestAuthHeader(tests.TestCase):
304
262
 
305
263
            protocol_version = 'HTTP/0.1'
306
264
 
307
 
        server = http_server.HttpServer(BogusRequestHandler)
308
 
        try:
309
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
310
 
        except:
311
 
            server.stop_server()
312
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
265
        self.assertRaises(httplib.UnknownProtocol,
 
266
                          http_server.HttpServer, BogusRequestHandler)
313
267
 
314
268
    def test_force_invalid_protocol(self):
315
 
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
316
 
        try:
317
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
318
 
        except:
319
 
            server.stop_server()
320
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
269
        self.assertRaises(httplib.UnknownProtocol,
 
270
                          http_server.HttpServer, protocol_version='HTTP/0.1')
321
271
 
322
272
    def test_server_start_and_stop(self):
323
273
        server = http_server.HttpServer()
 
274
        self.addCleanup(server.stop_server)
324
275
        server.start_server()
325
 
        try:
326
 
            self.assertTrue(server._http_running)
327
 
        finally:
328
 
            server.stop_server()
329
 
        self.assertFalse(server._http_running)
 
276
        self.assertTrue(server.server is not None)
 
277
        self.assertTrue(server.server.serving is not None)
 
278
        self.assertTrue(server.server.serving)
330
279
 
331
280
    def test_create_http_server_one_zero(self):
332
281
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
335
284
 
336
285
        server = http_server.HttpServer(RequestHandlerOneZero)
337
286
        self.start_server(server)
338
 
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
 
287
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
339
288
 
340
289
    def test_create_http_server_one_one(self):
341
290
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
344
293
 
345
294
        server = http_server.HttpServer(RequestHandlerOneOne)
346
295
        self.start_server(server)
347
 
        self.assertIsInstance(server._httpd,
 
296
        self.assertIsInstance(server.server,
348
297
                              http_server.TestingThreadingHTTPServer)
349
298
 
350
299
    def test_create_http_server_force_one_one(self):
355
304
        server = http_server.HttpServer(RequestHandlerOneZero,
356
305
                                        protocol_version='HTTP/1.1')
357
306
        self.start_server(server)
358
 
        self.assertIsInstance(server._httpd,
 
307
        self.assertIsInstance(server.server,
359
308
                              http_server.TestingThreadingHTTPServer)
360
309
 
361
310
    def test_create_http_server_force_one_zero(self):
366
315
        server = http_server.HttpServer(RequestHandlerOneOne,
367
316
                                        protocol_version='HTTP/1.0')
368
317
        self.start_server(server)
369
 
        self.assertIsInstance(server._httpd,
 
318
        self.assertIsInstance(server.server,
370
319
                              http_server.TestingHTTPServer)
371
320
 
372
321
 
401
350
class TestHttpTransportUrls(tests.TestCase):
402
351
    """Test the http urls."""
403
352
 
 
353
    scenarios = vary_by_http_client_implementation()
 
354
 
404
355
    def test_abs_url(self):
405
356
        """Construction of absolute http URLs"""
406
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
357
        t = self._transport('http://example.com/bzr/bzr.dev/')
407
358
        eq = self.assertEqualDiff
408
 
        eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
409
 
        eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
410
 
        eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
 
359
        eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
 
360
        eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
 
361
        eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
411
362
        eq(t.abspath('.bzr/1//2/./3'),
412
 
           'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
 
363
           'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
413
364
 
414
365
    def test_invalid_http_urls(self):
415
366
        """Trap invalid construction of urls"""
416
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
367
        self._transport('http://example.com/bzr/bzr.dev/')
417
368
        self.assertRaises(errors.InvalidURL,
418
369
                          self._transport,
419
 
                          'http://http://bazaar-vcs.org/bzr/bzr.dev/')
 
370
                          'http://http://example.com/bzr/bzr.dev/')
420
371
 
421
372
    def test_http_root_urls(self):
422
373
        """Construction of URLs from server root"""
423
 
        t = self._transport('http://bzr.ozlabs.org/')
 
374
        t = self._transport('http://example.com/')
424
375
        eq = self.assertEqualDiff
425
376
        eq(t.abspath('.bzr/tree-version'),
426
 
           'http://bzr.ozlabs.org/.bzr/tree-version')
 
377
           'http://example.com/.bzr/tree-version')
427
378
 
428
379
    def test_http_impl_urls(self):
429
380
        """There are servers which ask for particular clients to connect"""
431
382
        server.start_server()
432
383
        try:
433
384
            url = server.get_url()
434
 
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
 
385
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
435
386
        finally:
436
387
            server.stop_server()
437
388
 
475
426
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
476
427
    """Test the http connections."""
477
428
 
 
429
    scenarios = multiply_scenarios(
 
430
        vary_by_http_client_implementation(), 
 
431
        vary_by_http_protocol_version(),
 
432
        )
 
433
 
478
434
    def setUp(self):
479
435
        http_utils.TestCaseWithWebserver.setUp(self)
480
436
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
482
438
 
483
439
    def test_http_has(self):
484
440
        server = self.get_readonly_server()
485
 
        t = self._transport(server.get_url())
 
441
        t = self.get_readonly_transport()
486
442
        self.assertEqual(t.has('foo/bar'), True)
487
443
        self.assertEqual(len(server.logs), 1)
488
444
        self.assertContainsRe(server.logs[0],
490
446
 
491
447
    def test_http_has_not_found(self):
492
448
        server = self.get_readonly_server()
493
 
        t = self._transport(server.get_url())
 
449
        t = self.get_readonly_transport()
494
450
        self.assertEqual(t.has('not-found'), False)
495
451
        self.assertContainsRe(server.logs[1],
496
452
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
497
453
 
498
454
    def test_http_get(self):
499
455
        server = self.get_readonly_server()
500
 
        t = self._transport(server.get_url())
 
456
        t = self.get_readonly_transport()
501
457
        fp = t.get('foo/bar')
502
458
        self.assertEqualDiff(
503
459
            fp.read(),
525
481
class TestHttpTransportRegistration(tests.TestCase):
526
482
    """Test registrations of various http implementations"""
527
483
 
 
484
    scenarios = vary_by_http_client_implementation()
 
485
 
528
486
    def test_http_registered(self):
529
 
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
 
487
        t = transport.get_transport('%s://foo.com/' % self._url_protocol)
530
488
        self.assertIsInstance(t, transport.Transport)
531
489
        self.assertIsInstance(t, self._transport)
532
490
 
533
491
 
534
492
class TestPost(tests.TestCase):
535
493
 
 
494
    scenarios = multiply_scenarios(
 
495
        vary_by_http_client_implementation(), 
 
496
        vary_by_http_protocol_version(),
 
497
        )
 
498
 
536
499
    def test_post_body_is_received(self):
537
500
        server = RecordingServer(expect_body_tail='end-of-body',
538
 
            scheme=self._qualified_prefix)
 
501
                                 scheme=self._url_protocol)
539
502
        self.start_server(server)
540
503
        url = server.get_url()
541
 
        http_transport = self._transport(url)
 
504
        # FIXME: needs a cleanup -- vila 20100611
 
505
        http_transport = transport.get_transport(url)
542
506
        code, response = http_transport._post('abc def end-of-body')
543
507
        self.assertTrue(
544
508
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
545
509
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
 
510
        self.assertTrue('content-type: application/octet-stream\r'
 
511
                        in server.received_bytes.lower())
546
512
        # The transport should not be assuming that the server can accept
547
513
        # chunked encoding the first time it connects, because HTTP/1.1, so we
548
514
        # check for the literal string.
584
550
    Daughter classes are expected to override _req_handler_class
585
551
    """
586
552
 
 
553
    scenarios = multiply_scenarios(
 
554
        vary_by_http_client_implementation(), 
 
555
        vary_by_http_protocol_version(),
 
556
        )
 
557
 
587
558
    # Provide a useful default
588
559
    _req_handler_class = http_server.TestingHTTPRequestHandler
589
560
 
590
561
    def create_transport_readonly_server(self):
591
 
        return http_server.HttpServer(self._req_handler_class,
592
 
                                      protocol_version=self._protocol_version)
 
562
        server = http_server.HttpServer(self._req_handler_class,
 
563
                                        protocol_version=self._protocol_version)
 
564
        server._url_protocol = self._url_protocol
 
565
        return server
593
566
 
594
567
    def _testing_pycurl(self):
595
568
        # TODO: This is duplicated for lots of the classes in this file
600
573
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
601
574
    """Whatever request comes in, close the connection"""
602
575
 
603
 
    def handle_one_request(self):
 
576
    def _handle_one_request(self):
604
577
        """Handle a single HTTP request, by abruptly closing the connection"""
605
578
        self.close_connection = 1
606
579
 
611
584
    _req_handler_class = WallRequestHandler
612
585
 
613
586
    def test_http_has(self):
614
 
        server = self.get_readonly_server()
615
 
        t = self._transport(server.get_url())
 
587
        t = self.get_readonly_transport()
616
588
        # Unfortunately httplib (see HTTPResponse._read_status
617
589
        # for details) make no distinction between a closed
618
590
        # socket and badly formatted status line, so we can't
624
596
                          t.has, 'foo/bar')
625
597
 
626
598
    def test_http_get(self):
627
 
        server = self.get_readonly_server()
628
 
        t = self._transport(server.get_url())
 
599
        t = self.get_readonly_transport()
629
600
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
630
601
                           errors.InvalidHttpResponse),
631
602
                          t.get, 'foo/bar')
648
619
    _req_handler_class = BadStatusRequestHandler
649
620
 
650
621
    def test_http_has(self):
651
 
        server = self.get_readonly_server()
652
 
        t = self._transport(server.get_url())
 
622
        t = self.get_readonly_transport()
653
623
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
654
624
 
655
625
    def test_http_get(self):
656
 
        server = self.get_readonly_server()
657
 
        t = self._transport(server.get_url())
 
626
        t = self.get_readonly_transport()
658
627
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
659
628
 
660
629
 
665
634
        """Fakes handling a single HTTP request, returns a bad status"""
666
635
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
667
636
        self.wfile.write("Invalid status line\r\n")
 
637
        # If we don't close the connection pycurl will hang. Since this is a
 
638
        # stress test we don't *have* to respect the protocol, but we don't
 
639
        # have to sabotage it too much either.
 
640
        self.close_connection = True
668
641
        return False
669
642
 
670
643
 
676
649
 
677
650
    _req_handler_class = InvalidStatusRequestHandler
678
651
 
679
 
    def test_http_has(self):
680
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
681
 
            raise tests.KnownFailure(
682
 
                'pycurl hangs if the server send back garbage')
683
 
        super(TestInvalidStatusServer, self).test_http_has()
684
 
 
685
 
    def test_http_get(self):
686
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
687
 
            raise tests.KnownFailure(
688
 
                'pycurl hangs if the server send back garbage')
689
 
        super(TestInvalidStatusServer, self).test_http_get()
690
 
 
691
652
 
692
653
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
693
654
    """Whatever request comes in, returns a bad protocol version"""
715
676
        super(TestBadProtocolServer, self).setUp()
716
677
 
717
678
    def test_http_has(self):
718
 
        server = self.get_readonly_server()
719
 
        t = self._transport(server.get_url())
 
679
        t = self.get_readonly_transport()
720
680
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
721
681
 
722
682
    def test_http_get(self):
723
 
        server = self.get_readonly_server()
724
 
        t = self._transport(server.get_url())
 
683
        t = self.get_readonly_transport()
725
684
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
726
685
 
727
686
 
741
700
    _req_handler_class = ForbiddenRequestHandler
742
701
 
743
702
    def test_http_has(self):
744
 
        server = self.get_readonly_server()
745
 
        t = self._transport(server.get_url())
 
703
        t = self.get_readonly_transport()
746
704
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
747
705
 
748
706
    def test_http_get(self):
749
 
        server = self.get_readonly_server()
750
 
        t = self._transport(server.get_url())
 
707
        t = self.get_readonly_transport()
751
708
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
752
709
 
753
710
 
792
749
        self.build_tree_contents([('a', '0123456789')],)
793
750
 
794
751
    def test_readv(self):
795
 
        server = self.get_readonly_server()
796
 
        t = self._transport(server.get_url())
 
752
        t = self.get_readonly_transport()
797
753
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
798
754
        self.assertEqual(l[0], (0, '0'))
799
755
        self.assertEqual(l[1], (1, '1'))
801
757
        self.assertEqual(l[3], (9, '9'))
802
758
 
803
759
    def test_readv_out_of_order(self):
804
 
        server = self.get_readonly_server()
805
 
        t = self._transport(server.get_url())
 
760
        t = self.get_readonly_transport()
806
761
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
807
762
        self.assertEqual(l[0], (1, '1'))
808
763
        self.assertEqual(l[1], (9, '9'))
810
765
        self.assertEqual(l[3], (3, '34'))
811
766
 
812
767
    def test_readv_invalid_ranges(self):
813
 
        server = self.get_readonly_server()
814
 
        t = self._transport(server.get_url())
 
768
        t = self.get_readonly_transport()
815
769
 
816
770
        # This is intentionally reading off the end of the file
817
771
        # since we are sure that it cannot get there
825
779
 
826
780
    def test_readv_multiple_get_requests(self):
827
781
        server = self.get_readonly_server()
828
 
        t = self._transport(server.get_url())
 
782
        t = self.get_readonly_transport()
829
783
        # force transport to issue multiple requests
830
784
        t._max_readv_combine = 1
831
785
        t._max_get_ranges = 1
839
793
 
840
794
    def test_readv_get_max_size(self):
841
795
        server = self.get_readonly_server()
842
 
        t = self._transport(server.get_url())
 
796
        t = self.get_readonly_transport()
843
797
        # force transport to issue multiple requests by limiting the number of
844
798
        # bytes by request. Note that this apply to coalesced offsets only, a
845
799
        # single range will keep its size even if bigger than the limit.
854
808
 
855
809
    def test_complete_readv_leave_pipe_clean(self):
856
810
        server = self.get_readonly_server()
857
 
        t = self._transport(server.get_url())
 
811
        t = self.get_readonly_transport()
858
812
        # force transport to issue multiple requests
859
813
        t._get_max_size = 2
860
 
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
 
814
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
861
815
        # The server should have issued 3 requests
862
816
        self.assertEqual(3, server.GET_request_nb)
863
817
        self.assertEqual('0123456789', t.get_bytes('a'))
865
819
 
866
820
    def test_incomplete_readv_leave_pipe_clean(self):
867
821
        server = self.get_readonly_server()
868
 
        t = self._transport(server.get_url())
 
822
        t = self.get_readonly_transport()
869
823
        # force transport to issue multiple requests
870
824
        t._get_max_size = 2
871
825
        # Don't collapse readv results into a list so that we leave unread
940
894
    def get_multiple_ranges(self, file, file_size, ranges):
941
895
        self.send_response(206)
942
896
        self.send_header('Accept-Ranges', 'bytes')
 
897
        # XXX: this is strange; the 'random' name below seems undefined and
 
898
        # yet the tests pass -- mbp 2010-10-11 bug 658773
943
899
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
944
900
        self.send_header("Content-Type",
945
901
                         "multipart/byteranges; boundary=%s" % boundary)
1007
963
                return
1008
964
            self.send_range_content(file, start, end - start + 1)
1009
965
            cur += 1
1010
 
        # No final boundary
 
966
        # Final boundary
1011
967
        self.wfile.write(boundary_line)
1012
968
 
1013
969
 
1021
977
 
1022
978
    def test_readv_with_short_reads(self):
1023
979
        server = self.get_readonly_server()
1024
 
        t = self._transport(server.get_url())
 
980
        t = self.get_readonly_transport()
1025
981
        # Force separate ranges for each offset
1026
982
        t._bytes_to_read_before_seek = 0
1027
983
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1042
998
        # that mode
1043
999
        self.assertEqual('single', t._range_hint)
1044
1000
 
 
1001
 
1045
1002
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1046
1003
    """Errors out when range specifiers exceed the limit"""
1047
1004
 
1071
1028
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1072
1029
    """Tests readv requests against a server erroring out on too much ranges."""
1073
1030
 
 
1031
    scenarios = multiply_scenarios(
 
1032
        vary_by_http_client_implementation(), 
 
1033
        vary_by_http_protocol_version(),
 
1034
        )
 
1035
 
1074
1036
    # Requests with more range specifiers will error out
1075
1037
    range_limit = 3
1076
1038
 
1078
1040
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1079
1041
                                      protocol_version=self._protocol_version)
1080
1042
 
1081
 
    def get_transport(self):
1082
 
        return self._transport(self.get_readonly_server().get_url())
1083
 
 
1084
1043
    def setUp(self):
1085
1044
        http_utils.TestCaseWithWebserver.setUp(self)
1086
1045
        # We need to manipulate ranges that correspond to real chunks in the
1090
1049
        self.build_tree_contents([('a', content)],)
1091
1050
 
1092
1051
    def test_few_ranges(self):
1093
 
        t = self.get_transport()
 
1052
        t = self.get_readonly_transport()
1094
1053
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1095
1054
        self.assertEqual(l[0], (0, '0000'))
1096
1055
        self.assertEqual(l[1], (1024, '0001'))
1097
1056
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1098
1057
 
1099
1058
    def test_more_ranges(self):
1100
 
        t = self.get_transport()
 
1059
        t = self.get_readonly_transport()
1101
1060
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1102
1061
        self.assertEqual(l[0], (0, '0000'))
1103
1062
        self.assertEqual(l[1], (1024, '0001'))
1114
1073
    Only the urllib implementation is tested here.
1115
1074
    """
1116
1075
 
1117
 
    def setUp(self):
1118
 
        tests.TestCase.setUp(self)
1119
 
        self._old_env = {}
1120
 
 
1121
 
    def tearDown(self):
1122
 
        self._restore_env()
1123
 
        tests.TestCase.tearDown(self)
1124
 
 
1125
 
    def _install_env(self, env):
1126
 
        for name, value in env.iteritems():
1127
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1128
 
 
1129
 
    def _restore_env(self):
1130
 
        for name, value in self._old_env.iteritems():
1131
 
            osutils.set_or_unset_env(name, value)
1132
 
 
1133
1076
    def _proxied_request(self):
1134
1077
        handler = _urllib2_wrappers.ProxyHandler()
1135
1078
        request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
1137
1080
        return request
1138
1081
 
1139
1082
    def test_empty_user(self):
1140
 
        self._install_env({'http_proxy': 'http://bar.com'})
 
1083
        self.overrideEnv('http_proxy', 'http://bar.com')
1141
1084
        request = self._proxied_request()
1142
1085
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1143
1086
 
1144
1087
    def test_invalid_proxy(self):
1145
1088
        """A proxy env variable without scheme"""
1146
 
        self._install_env({'http_proxy': 'host:1234'})
 
1089
        self.overrideEnv('http_proxy', 'host:1234')
1147
1090
        self.assertRaises(errors.InvalidURL, self._proxied_request)
1148
1091
 
1149
1092
 
1156
1099
    to the file names).
1157
1100
    """
1158
1101
 
 
1102
    scenarios = multiply_scenarios(
 
1103
        vary_by_http_client_implementation(), 
 
1104
        vary_by_http_protocol_version(),
 
1105
        )
 
1106
 
1159
1107
    # FIXME: We don't have an https server available, so we don't
1160
 
    # test https connections.
 
1108
    # test https connections. --vila toolongago
1161
1109
 
1162
1110
    def setUp(self):
1163
1111
        super(TestProxyHttpServer, self).setUp()
 
1112
        self.transport_secondary_server = http_utils.ProxyServer
1164
1113
        self.build_tree_contents([('foo', 'contents of foo\n'),
1165
1114
                                  ('foo-proxied', 'proxied contents of foo\n')])
1166
1115
        # Let's setup some attributes for tests
1167
 
        self.server = self.get_readonly_server()
1168
 
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
 
1116
        server = self.get_readonly_server()
 
1117
        self.server_host_port = '%s:%d' % (server.host, server.port)
1169
1118
        if self._testing_pycurl():
1170
1119
            # Oh my ! pycurl does not check for the port as part of
1171
1120
            # no_proxy :-( So we just test the host part
1172
 
            self.no_proxy_host = self.server.host
 
1121
            self.no_proxy_host = server.host
1173
1122
        else:
1174
 
            self.no_proxy_host = self.proxy_address
 
1123
            self.no_proxy_host = self.server_host_port
1175
1124
        # The secondary server is the proxy
1176
 
        self.proxy = self.get_secondary_server()
1177
 
        self.proxy_url = self.proxy.get_url()
1178
 
        self._old_env = {}
 
1125
        self.proxy_url = self.get_secondary_url()
1179
1126
 
1180
1127
    def _testing_pycurl(self):
1181
1128
        # TODO: This is duplicated for lots of the classes in this file
1182
1129
        return (features.pycurl.available()
1183
1130
                and self._transport == PyCurlTransport)
1184
1131
 
1185
 
    def create_transport_secondary_server(self):
1186
 
        """Creates an http server that will serve files with
1187
 
        '-proxied' appended to their names.
1188
 
        """
1189
 
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
1190
 
 
1191
 
    def _install_env(self, env):
1192
 
        for name, value in env.iteritems():
1193
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1194
 
 
1195
 
    def _restore_env(self):
1196
 
        for name, value in self._old_env.iteritems():
1197
 
            osutils.set_or_unset_env(name, value)
1198
 
 
1199
 
    def proxied_in_env(self, env):
1200
 
        self._install_env(env)
1201
 
        url = self.server.get_url()
1202
 
        t = self._transport(url)
1203
 
        try:
1204
 
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1205
 
        finally:
1206
 
            self._restore_env()
1207
 
 
1208
 
    def not_proxied_in_env(self, env):
1209
 
        self._install_env(env)
1210
 
        url = self.server.get_url()
1211
 
        t = self._transport(url)
1212
 
        try:
1213
 
            self.assertEqual('contents of foo\n', t.get('foo').read())
1214
 
        finally:
1215
 
            self._restore_env()
 
1132
    def assertProxied(self):
 
1133
        t = self.get_readonly_transport()
 
1134
        self.assertEqual('proxied contents of foo\n', t.get('foo').read())
 
1135
 
 
1136
    def assertNotProxied(self):
 
1137
        t = self.get_readonly_transport()
 
1138
        self.assertEqual('contents of foo\n', t.get('foo').read())
1216
1139
 
1217
1140
    def test_http_proxy(self):
1218
 
        self.proxied_in_env({'http_proxy': self.proxy_url})
 
1141
        self.overrideEnv('http_proxy', self.proxy_url)
 
1142
        self.assertProxied()
1219
1143
 
1220
1144
    def test_HTTP_PROXY(self):
1221
1145
        if self._testing_pycurl():
1224
1148
            # about. Should we ?)
1225
1149
            raise tests.TestNotApplicable(
1226
1150
                'pycurl does not check HTTP_PROXY for security reasons')
1227
 
        self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
 
1151
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1152
        self.assertProxied()
1228
1153
 
1229
1154
    def test_all_proxy(self):
1230
 
        self.proxied_in_env({'all_proxy': self.proxy_url})
 
1155
        self.overrideEnv('all_proxy', self.proxy_url)
 
1156
        self.assertProxied()
1231
1157
 
1232
1158
    def test_ALL_PROXY(self):
1233
 
        self.proxied_in_env({'ALL_PROXY': self.proxy_url})
 
1159
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1160
        self.assertProxied()
1234
1161
 
1235
1162
    def test_http_proxy_with_no_proxy(self):
1236
 
        self.not_proxied_in_env({'http_proxy': self.proxy_url,
1237
 
                                 'no_proxy': self.no_proxy_host})
 
1163
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1164
        self.overrideEnv('http_proxy', self.proxy_url)
 
1165
        self.assertNotProxied()
1238
1166
 
1239
1167
    def test_HTTP_PROXY_with_NO_PROXY(self):
1240
1168
        if self._testing_pycurl():
1241
1169
            raise tests.TestNotApplicable(
1242
1170
                'pycurl does not check HTTP_PROXY for security reasons')
1243
 
        self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
1244
 
                                 'NO_PROXY': self.no_proxy_host})
 
1171
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1172
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1173
        self.assertNotProxied()
1245
1174
 
1246
1175
    def test_all_proxy_with_no_proxy(self):
1247
 
        self.not_proxied_in_env({'all_proxy': self.proxy_url,
1248
 
                                 'no_proxy': self.no_proxy_host})
 
1176
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1177
        self.overrideEnv('all_proxy', self.proxy_url)
 
1178
        self.assertNotProxied()
1249
1179
 
1250
1180
    def test_ALL_PROXY_with_NO_PROXY(self):
1251
 
        self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
1252
 
                                 'NO_PROXY': self.no_proxy_host})
 
1181
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1182
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1183
        self.assertNotProxied()
1253
1184
 
1254
1185
    def test_http_proxy_without_scheme(self):
 
1186
        self.overrideEnv('http_proxy', self.server_host_port)
1255
1187
        if self._testing_pycurl():
1256
1188
            # pycurl *ignores* invalid proxy env variables. If that ever change
1257
1189
            # in the future, this test will fail indicating that pycurl do not
1258
1190
            # ignore anymore such variables.
1259
 
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
 
1191
            self.assertNotProxied()
1260
1192
        else:
1261
 
            self.assertRaises(errors.InvalidURL,
1262
 
                              self.proxied_in_env,
1263
 
                              {'http_proxy': self.proxy_address})
 
1193
            self.assertRaises(errors.InvalidURL, self.assertProxied)
1264
1194
 
1265
1195
 
1266
1196
class TestRanges(http_utils.TestCaseWithWebserver):
1267
1197
    """Test the Range header in GET methods."""
1268
1198
 
 
1199
    scenarios = multiply_scenarios(
 
1200
        vary_by_http_client_implementation(), 
 
1201
        vary_by_http_protocol_version(),
 
1202
        )
 
1203
 
1269
1204
    def setUp(self):
1270
1205
        http_utils.TestCaseWithWebserver.setUp(self)
1271
1206
        self.build_tree_contents([('a', '0123456789')],)
1272
 
        server = self.get_readonly_server()
1273
 
        self.transport = self._transport(server.get_url())
1274
1207
 
1275
1208
    def create_transport_readonly_server(self):
1276
1209
        return http_server.HttpServer(protocol_version=self._protocol_version)
1277
1210
 
1278
1211
    def _file_contents(self, relpath, ranges):
 
1212
        t = self.get_readonly_transport()
1279
1213
        offsets = [ (start, end - start + 1) for start, end in ranges]
1280
 
        coalesce = self.transport._coalesce_offsets
 
1214
        coalesce = t._coalesce_offsets
1281
1215
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1282
 
        code, data = self.transport._get(relpath, coalesced)
 
1216
        code, data = t._get(relpath, coalesced)
1283
1217
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1284
1218
        for start, end in ranges:
1285
1219
            data.seek(start)
1286
1220
            yield data.read(end - start + 1)
1287
1221
 
1288
1222
    def _file_tail(self, relpath, tail_amount):
1289
 
        code, data = self.transport._get(relpath, [], tail_amount)
 
1223
        t = self.get_readonly_transport()
 
1224
        code, data = t._get(relpath, [], tail_amount)
1290
1225
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1291
1226
        data.seek(-tail_amount, 2)
1292
1227
        return data.read(tail_amount)
1311
1246
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1312
1247
    """Test redirection between http servers."""
1313
1248
 
1314
 
    def create_transport_secondary_server(self):
1315
 
        """Create the secondary server redirecting to the primary server"""
1316
 
        new = self.get_readonly_server()
1317
 
 
1318
 
        redirecting = http_utils.HTTPServerRedirecting(
1319
 
            protocol_version=self._protocol_version)
1320
 
        redirecting.redirect_to(new.host, new.port)
1321
 
        return redirecting
 
1249
    scenarios = multiply_scenarios(
 
1250
        vary_by_http_client_implementation(), 
 
1251
        vary_by_http_protocol_version(),
 
1252
        )
1322
1253
 
1323
1254
    def setUp(self):
1324
1255
        super(TestHTTPRedirections, self).setUp()
1326
1257
                                  ('bundle',
1327
1258
                                  '# Bazaar revision bundle v0.9\n#\n')
1328
1259
                                  ],)
1329
 
        # The requests to the old server will be redirected to the new server
1330
 
        self.old_transport = self._transport(self.old_server.get_url())
1331
1260
 
1332
1261
    def test_redirected(self):
1333
 
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1334
 
        t = self._transport(self.new_server.get_url())
1335
 
        self.assertEqual('0123456789', t.get('a').read())
1336
 
 
1337
 
    def test_read_redirected_bundle_from_url(self):
1338
 
        from bzrlib.bundle import read_bundle_from_url
1339
 
        url = self.old_transport.abspath('bundle')
1340
 
        bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
1341
 
                read_bundle_from_url, url)
1342
 
        # If read_bundle_from_url was successful we get an empty bundle
1343
 
        self.assertEqual([], bundle.revisions)
 
1262
        self.assertRaises(errors.RedirectRequested,
 
1263
                          self.get_old_transport().get, 'a')
 
1264
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1344
1265
 
1345
1266
 
1346
1267
class RedirectedRequest(_urllib2_wrappers.Request):
1363
1284
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1364
1285
 
1365
1286
 
 
1287
def cleanup_http_redirection_connections(test):
 
1288
    # Some sockets are opened but never seen by _urllib, so we trap them at
 
1289
    # the _urllib2_wrappers level to be able to clean them up.
 
1290
    def socket_disconnect(sock):
 
1291
        try:
 
1292
            sock.shutdown(socket.SHUT_RDWR)
 
1293
            sock.close()
 
1294
        except socket.error:
 
1295
            pass
 
1296
    def connect(connection):
 
1297
        test.http_connect_orig(connection)
 
1298
        test.addCleanup(socket_disconnect, connection.sock)
 
1299
    test.http_connect_orig = test.overrideAttr(
 
1300
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
 
1301
    def connect(connection):
 
1302
        test.https_connect_orig(connection)
 
1303
        test.addCleanup(socket_disconnect, connection.sock)
 
1304
    test.https_connect_orig = test.overrideAttr(
 
1305
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
 
1306
 
 
1307
 
1366
1308
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1367
1309
    """Test redirections.
1368
1310
 
1377
1319
    -- vila 20070212
1378
1320
    """
1379
1321
 
 
1322
    scenarios = multiply_scenarios(
 
1323
        vary_by_http_client_implementation(), 
 
1324
        vary_by_http_protocol_version(),
 
1325
        )
 
1326
 
1380
1327
    def setUp(self):
1381
1328
        if (features.pycurl.available()
1382
1329
            and self._transport == PyCurlTransport):
1383
1330
            raise tests.TestNotApplicable(
1384
 
                "pycurl doesn't redirect silently annymore")
 
1331
                "pycurl doesn't redirect silently anymore")
1385
1332
        super(TestHTTPSilentRedirections, self).setUp()
1386
1333
        install_redirected_request(self)
 
1334
        cleanup_http_redirection_connections(self)
1387
1335
        self.build_tree_contents([('a','a'),
1388
1336
                                  ('1/',),
1389
1337
                                  ('1/a', 'redirected once'),
1397
1345
                                  ('5/a', 'redirected 5 times'),
1398
1346
                                  ],)
1399
1347
 
1400
 
        self.old_transport = self._transport(self.old_server.get_url())
1401
 
 
1402
 
    def create_transport_secondary_server(self):
1403
 
        """Create the secondary server, redirections are defined in the tests"""
1404
 
        return http_utils.HTTPServerRedirecting(
1405
 
            protocol_version=self._protocol_version)
1406
 
 
1407
1348
    def test_one_redirection(self):
1408
 
        t = self.old_transport
1409
 
 
1410
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1349
        t = self.get_old_transport()
 
1350
        req = RedirectedRequest('GET', t._remote_path('a'))
1411
1351
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1412
1352
                                       self.new_server.port)
1413
1353
        self.old_server.redirections = \
1414
1354
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1415
 
        self.assertEqual('redirected once',t._perform(req).read())
 
1355
        self.assertEqual('redirected once', t._perform(req).read())
1416
1356
 
1417
1357
    def test_five_redirections(self):
1418
 
        t = self.old_transport
1419
 
 
1420
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1358
        t = self.get_old_transport()
 
1359
        req = RedirectedRequest('GET', t._remote_path('a'))
1421
1360
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1422
1361
                                       self.old_server.port)
1423
1362
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1429
1368
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1430
1369
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1431
1370
            ]
1432
 
        self.assertEqual('redirected 5 times',t._perform(req).read())
 
1371
        self.assertEqual('redirected 5 times', t._perform(req).read())
1433
1372
 
1434
1373
 
1435
1374
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1436
1375
    """Test transport.do_catching_redirections."""
1437
1376
 
 
1377
    scenarios = multiply_scenarios(
 
1378
        vary_by_http_client_implementation(), 
 
1379
        vary_by_http_protocol_version(),
 
1380
        )
 
1381
 
1438
1382
    def setUp(self):
1439
1383
        super(TestDoCatchRedirections, self).setUp()
1440
1384
        self.build_tree_contents([('a', '0123456789'),],)
1441
 
 
1442
 
        self.old_transport = self._transport(self.old_server.get_url())
1443
 
 
1444
 
    def get_a(self, transport):
1445
 
        return transport.get('a')
 
1385
        cleanup_http_redirection_connections(self)
 
1386
 
 
1387
        self.old_transport = self.get_old_transport()
 
1388
 
 
1389
    def get_a(self, t):
 
1390
        return t.get('a')
1446
1391
 
1447
1392
    def test_no_redirection(self):
1448
 
        t = self._transport(self.new_server.get_url())
 
1393
        t = self.get_new_transport()
1449
1394
 
1450
1395
        # We use None for redirected so that we fail if redirected
1451
1396
        self.assertEqual('0123456789',
1455
1400
    def test_one_redirection(self):
1456
1401
        self.redirections = 0
1457
1402
 
1458
 
        def redirected(transport, exception, redirection_notice):
 
1403
        def redirected(t, exception, redirection_notice):
1459
1404
            self.redirections += 1
1460
 
            dir, file = urlutils.split(exception.target)
1461
 
            return self._transport(dir)
 
1405
            redirected_t = t._redirected_to(exception.source, exception.target)
 
1406
            return redirected_t
1462
1407
 
1463
1408
        self.assertEqual('0123456789',
1464
1409
                         transport.do_catching_redirections(
1481
1426
class TestAuth(http_utils.TestCaseWithWebserver):
1482
1427
    """Test authentication scheme"""
1483
1428
 
 
1429
    scenarios = multiply_scenarios(
 
1430
        vary_by_http_client_implementation(),
 
1431
        vary_by_http_protocol_version(),
 
1432
        vary_by_http_auth_scheme(),
 
1433
        )
 
1434
 
1484
1435
    _auth_header = 'Authorization'
1485
1436
    _password_prompt_prefix = ''
1486
1437
    _username_prompt_prefix = ''
1494
1445
                                  ('b', 'contents of b\n'),])
1495
1446
 
1496
1447
    def create_transport_readonly_server(self):
1497
 
        return self._auth_server(protocol_version=self._protocol_version)
 
1448
        server = self._auth_server(protocol_version=self._protocol_version)
 
1449
        server._url_protocol = self._url_protocol
 
1450
        return server
1498
1451
 
1499
1452
    def _testing_pycurl(self):
1500
1453
        # TODO: This is duplicated for lots of the classes in this file
1513
1466
        return url
1514
1467
 
1515
1468
    def get_user_transport(self, user, password):
1516
 
        return self._transport(self.get_user_url(user, password))
 
1469
        t = transport.get_transport(self.get_user_url(user, password))
 
1470
        return t
1517
1471
 
1518
1472
    def test_no_user(self):
1519
1473
        self.server.add_user('joe', 'foo')
1630
1584
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1631
1585
                                            stderr=tests.StringIOWrapper())
1632
1586
        # Create a minimal config file with the right password
1633
 
        conf = config.AuthenticationConfig()
1634
 
        conf._get_config().update(
1635
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1636
 
                          'user': user, 'password': password}})
1637
 
        conf._save()
 
1587
        _setup_authentication_config(
 
1588
            scheme='http', 
 
1589
            port=self.server.port,
 
1590
            user=user,
 
1591
            password=password)
1638
1592
        # Issue a request to the server to connect
1639
1593
        self.assertEqual('contents of a\n',t.get('a').read())
1640
1594
        # stdin should have  been left untouched
1642
1596
        # Only one 'Authentication Required' error should occur
1643
1597
        self.assertEqual(1, self.server.auth_required_errors)
1644
1598
 
1645
 
    def test_user_from_auth_conf(self):
1646
 
        if self._testing_pycurl():
1647
 
            raise tests.TestNotApplicable(
1648
 
                'pycurl does not support authentication.conf')
1649
 
        user = 'joe'
1650
 
        password = 'foo'
1651
 
        self.server.add_user(user, password)
1652
 
        # Create a minimal config file with the right password
1653
 
        conf = config.AuthenticationConfig()
1654
 
        conf._get_config().update(
1655
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1656
 
                          'user': user, 'password': password}})
1657
 
        conf._save()
1658
 
        t = self.get_user_transport(None, None)
1659
 
        # Issue a request to the server to connect
1660
 
        self.assertEqual('contents of a\n', t.get('a').read())
1661
 
        # Only one 'Authentication Required' error should occur
1662
 
        self.assertEqual(1, self.server.auth_required_errors)
1663
 
 
1664
1599
    def test_changing_nonce(self):
1665
1600
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1666
1601
                                     http_utils.ProxyDigestAuthServer):
1682
1617
        # initial 'who are you' and a second 'who are you' with the new nonce)
1683
1618
        self.assertEqual(2, self.server.auth_required_errors)
1684
1619
 
 
1620
    def test_user_from_auth_conf(self):
 
1621
        if self._testing_pycurl():
 
1622
            raise tests.TestNotApplicable(
 
1623
                'pycurl does not support authentication.conf')
 
1624
        user = 'joe'
 
1625
        password = 'foo'
 
1626
        self.server.add_user(user, password)
 
1627
        _setup_authentication_config(
 
1628
            scheme='http', 
 
1629
            port=self.server.port,
 
1630
            user=user,
 
1631
            password=password)
 
1632
        t = self.get_user_transport(None, None)
 
1633
        # Issue a request to the server to connect
 
1634
        self.assertEqual('contents of a\n', t.get('a').read())
 
1635
        # Only one 'Authentication Required' error should occur
 
1636
        self.assertEqual(1, self.server.auth_required_errors)
 
1637
 
 
1638
 
 
1639
def _setup_authentication_config(**kwargs):
 
1640
    conf = config.AuthenticationConfig()
 
1641
    conf._get_config().update({'httptest': kwargs})
 
1642
    conf._save()
 
1643
 
 
1644
 
 
1645
 
 
1646
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
 
1647
    """Unit tests for glue by which urllib2 asks us for authentication"""
 
1648
 
 
1649
    def test_get_user_password_without_port(self):
 
1650
        """We cope if urllib2 doesn't tell us the port.
 
1651
 
 
1652
        See https://bugs.launchpad.net/bzr/+bug/654684
 
1653
        """
 
1654
        user = 'joe'
 
1655
        password = 'foo'
 
1656
        _setup_authentication_config(
 
1657
            scheme='http', 
 
1658
            host='localhost',
 
1659
            user=user,
 
1660
            password=password)
 
1661
        handler = _urllib2_wrappers.HTTPAuthHandler()
 
1662
        got_pass = handler.get_user_password(dict(
 
1663
            user='joe',
 
1664
            protocol='http',
 
1665
            host='localhost',
 
1666
            path='/',
 
1667
            realm='Realm',
 
1668
            ))
 
1669
        self.assertEquals((user, password), got_pass)
1685
1670
 
1686
1671
 
1687
1672
class TestProxyAuth(TestAuth):
1688
1673
    """Test proxy authentication schemes."""
1689
1674
 
 
1675
    scenarios = multiply_scenarios(
 
1676
        vary_by_http_client_implementation(),
 
1677
        vary_by_http_protocol_version(),
 
1678
        vary_by_http_proxy_auth_scheme(),
 
1679
        )
 
1680
 
1690
1681
    _auth_header = 'Proxy-authorization'
1691
1682
    _password_prompt_prefix = 'Proxy '
1692
1683
    _username_prompt_prefix = 'Proxy '
1693
1684
 
1694
1685
    def setUp(self):
1695
1686
        super(TestProxyAuth, self).setUp()
1696
 
        self._old_env = {}
1697
 
        self.addCleanup(self._restore_env)
1698
1687
        # Override the contents to avoid false positives
1699
1688
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1700
1689
                                  ('b', 'not proxied contents of b\n'),
1703
1692
                                  ])
1704
1693
 
1705
1694
    def get_user_transport(self, user, password):
1706
 
        self._install_env({'all_proxy': self.get_user_url(user, password)})
1707
 
        return self._transport(self.server.get_url())
1708
 
 
1709
 
    def _install_env(self, env):
1710
 
        for name, value in env.iteritems():
1711
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1712
 
 
1713
 
    def _restore_env(self):
1714
 
        for name, value in self._old_env.iteritems():
1715
 
            osutils.set_or_unset_env(name, value)
 
1695
        self.overrideEnv('all_proxy', self.get_user_url(user, password))
 
1696
        return TestAuth.get_user_transport(self, user, password)
1716
1697
 
1717
1698
    def test_empty_pass(self):
1718
1699
        if self._testing_pycurl():
1737
1718
        self.readfile = StringIO(socket_read_content)
1738
1719
        self.writefile = StringIO()
1739
1720
        self.writefile.close = lambda: None
 
1721
        self.close = lambda: None
1740
1722
 
1741
1723
    def makefile(self, mode='r', bufsize=None):
1742
1724
        if 'r' in mode:
1747
1729
 
1748
1730
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1749
1731
 
 
1732
    scenarios = multiply_scenarios(
 
1733
        vary_by_http_client_implementation(), 
 
1734
        vary_by_http_protocol_version(),
 
1735
        )
 
1736
 
1750
1737
    def setUp(self):
1751
1738
        super(SmartHTTPTunnellingTest, self).setUp()
1752
1739
        # We use the VFS layer as part of HTTP tunnelling tests.
1753
 
        self._captureVar('BZR_NO_SMART_VFS', None)
 
1740
        self.overrideEnv('BZR_NO_SMART_VFS', None)
1754
1741
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
 
1742
        self.http_server = self.get_readonly_server()
1755
1743
 
1756
1744
    def create_transport_readonly_server(self):
1757
 
        return http_utils.HTTPServerWithSmarts(
 
1745
        server = http_utils.HTTPServerWithSmarts(
1758
1746
            protocol_version=self._protocol_version)
 
1747
        server._url_protocol = self._url_protocol
 
1748
        return server
1759
1749
 
1760
1750
    def test_open_bzrdir(self):
1761
1751
        branch = self.make_branch('relpath')
1762
 
        http_server = self.get_readonly_server()
1763
 
        url = http_server.get_url() + 'relpath'
 
1752
        url = self.http_server.get_url() + 'relpath'
1764
1753
        bd = bzrdir.BzrDir.open(url)
 
1754
        self.addCleanup(bd.transport.disconnect)
1765
1755
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1766
1756
 
1767
1757
    def test_bulk_data(self):
1769
1759
        # The 'readv' command in the smart protocol both sends and receives
1770
1760
        # bulk data, so we use that.
1771
1761
        self.build_tree(['data-file'])
1772
 
        http_server = self.get_readonly_server()
1773
 
        http_transport = self._transport(http_server.get_url())
 
1762
        http_transport = transport.get_transport(self.http_server.get_url())
1774
1763
        medium = http_transport.get_smart_medium()
1775
1764
        # Since we provide the medium, the url below will be mostly ignored
1776
1765
        # during the test, as long as the path is '/'.
1784
1773
        post_body = 'hello\n'
1785
1774
        expected_reply_body = 'ok\x012\n'
1786
1775
 
1787
 
        http_server = self.get_readonly_server()
1788
 
        http_transport = self._transport(http_server.get_url())
 
1776
        http_transport = transport.get_transport(self.http_server.get_url())
1789
1777
        medium = http_transport.get_smart_medium()
1790
1778
        response = medium.send_http_smart_request(post_body)
1791
1779
        reply_body = response.read()
1792
1780
        self.assertEqual(expected_reply_body, reply_body)
1793
1781
 
1794
1782
    def test_smart_http_server_post_request_handler(self):
1795
 
        httpd = self.get_readonly_server()._get_httpd()
 
1783
        httpd = self.http_server.server
1796
1784
 
1797
1785
        socket = SampleSocket(
1798
1786
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1830
1818
 
1831
1819
    def test_probe_smart_server(self):
1832
1820
        """Test error handling against server refusing smart requests."""
1833
 
        server = self.get_readonly_server()
1834
 
        t = self._transport(server.get_url())
 
1821
        t = self.get_readonly_transport()
1835
1822
        # No need to build a valid smart request here, the server will not even
1836
1823
        # try to interpret it.
1837
1824
        self.assertRaises(errors.SmartProtocolError,
1838
1825
                          t.get_smart_medium().send_http_smart_request,
1839
1826
                          'whatever')
1840
1827
 
 
1828
 
1841
1829
class Test_redirected_to(tests.TestCase):
1842
1830
 
 
1831
    scenarios = vary_by_http_client_implementation()
 
1832
 
1843
1833
    def test_redirected_to_subdir(self):
1844
1834
        t = self._transport('http://www.example.com/foo')
1845
1835
        r = t._redirected_to('http://www.example.com/foo',
1896
1886
    line.
1897
1887
    """
1898
1888
 
1899
 
    def handle_one_request(self):
 
1889
    def _handle_one_request(self):
1900
1890
        tcs = self.server.test_case_server
1901
1891
        requestline = self.rfile.readline()
1902
1892
        headers = self.MessageClass(self.rfile, 0)
1966
1956
        # We override at class level because constructors may propagate the
1967
1957
        # bound method and render instance overriding ineffective (an
1968
1958
        # alternative would be to define a specific ui factory instead...)
1969
 
        self.orig_report_activity = self._transport._report_activity
1970
 
        self._transport._report_activity = report_activity
1971
 
 
1972
 
    def tearDown(self):
1973
 
        self._transport._report_activity = self.orig_report_activity
1974
 
        self.server.stop_server()
1975
 
        tests.TestCase.tearDown(self)
 
1959
        self.overrideAttr(self._transport, '_report_activity', report_activity)
 
1960
        self.addCleanup(self.server.stop_server)
1976
1961
 
1977
1962
    def get_transport(self):
1978
 
        return self._transport(self.server.get_url())
 
1963
        t = self._transport(self.server.get_url())
 
1964
        # FIXME: Needs cleanup -- vila 20100611
 
1965
        return t
1979
1966
 
1980
1967
    def assertActivitiesMatch(self):
1981
1968
        self.assertEqual(self.server.bytes_read,
2086
2073
'''
2087
2074
        t = self.get_transport()
2088
2075
        # We must send a single line of body bytes, see
2089
 
        # PredefinedRequestHandler.handle_one_request
 
2076
        # PredefinedRequestHandler._handle_one_request
2090
2077
        code, f = t._post('abc def end-of-body\n')
2091
2078
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2092
2079
        self.assertActivitiesMatch()
2094
2081
 
2095
2082
class TestActivity(tests.TestCase, TestActivityMixin):
2096
2083
 
 
2084
    scenarios = multiply_scenarios(
 
2085
        vary_by_http_activity(),
 
2086
        vary_by_http_protocol_version(),
 
2087
        )
 
2088
 
2097
2089
    def setUp(self):
2098
 
        tests.TestCase.setUp(self)
2099
 
        self.server = self._activity_server(self._protocol_version)
2100
 
        self.server.start_server()
2101
 
        self.activities = {}
2102
 
        def report_activity(t, bytes, direction):
2103
 
            count = self.activities.get(direction, 0)
2104
 
            count += bytes
2105
 
            self.activities[direction] = count
2106
 
 
2107
 
        # We override at class level because constructors may propagate the
2108
 
        # bound method and render instance overriding ineffective (an
2109
 
        # alternative would be to define a specific ui factory instead...)
2110
 
        self.orig_report_activity = self._transport._report_activity
2111
 
        self._transport._report_activity = report_activity
2112
 
 
2113
 
    def tearDown(self):
2114
 
        self._transport._report_activity = self.orig_report_activity
2115
 
        self.server.stop_server()
2116
 
        tests.TestCase.tearDown(self)
 
2090
        TestActivityMixin.setUp(self)
2117
2091
 
2118
2092
 
2119
2093
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2120
2094
 
 
2095
    # Unlike TestActivity, we are really testing ReportingFileSocket and
 
2096
    # ReportingSocket, so we don't need all the parametrization. Since
 
2097
    # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
 
2098
    # test them through their use by the transport than directly (that's a
 
2099
    # bit less clean but far more simpler and effective).
 
2100
    _activity_server = ActivityHTTPServer
 
2101
    _protocol_version = 'HTTP/1.1'
 
2102
 
2121
2103
    def setUp(self):
2122
 
        tests.TestCase.setUp(self)
2123
 
        # Unlike TestActivity, we are really testing ReportingFileSocket and
2124
 
        # ReportingSocket, so we don't need all the parametrization. Since
2125
 
        # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2126
 
        # test them through their use by the transport than directly (that's a
2127
 
        # bit less clean but far more simpler and effective).
2128
 
        self.server = ActivityHTTPServer('HTTP/1.1')
2129
 
        self._transport=_urllib.HttpTransport_urllib
2130
 
 
2131
 
        self.server.start_server()
2132
 
 
2133
 
        # We override at class level because constructors may propagate the
2134
 
        # bound method and render instance overriding ineffective (an
2135
 
        # alternative would be to define a specific ui factory instead...)
2136
 
        self.orig_report_activity = self._transport._report_activity
2137
 
        self._transport._report_activity = None
2138
 
 
2139
 
    def tearDown(self):
2140
 
        self._transport._report_activity = self.orig_report_activity
2141
 
        self.server.stop_server()
2142
 
        tests.TestCase.tearDown(self)
 
2104
        self._transport =_urllib.HttpTransport_urllib
 
2105
        TestActivityMixin.setUp(self)
2143
2106
 
2144
2107
    def assertActivitiesMatch(self):
2145
2108
        # Nothing to check here
2149
2112
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2150
2113
    """Test authentication on the redirected http server."""
2151
2114
 
 
2115
    scenarios = vary_by_http_protocol_version()
 
2116
 
2152
2117
    _auth_header = 'Authorization'
2153
2118
    _password_prompt_prefix = ''
2154
2119
    _username_prompt_prefix = ''
2155
2120
    _auth_server = http_utils.HTTPBasicAuthServer
2156
2121
    _transport = _urllib.HttpTransport_urllib
2157
2122
 
2158
 
    def create_transport_readonly_server(self):
2159
 
        return self._auth_server()
2160
 
 
2161
 
    def create_transport_secondary_server(self):
2162
 
        """Create the secondary server redirecting to the primary server"""
2163
 
        new = self.get_readonly_server()
2164
 
 
2165
 
        redirecting = http_utils.HTTPServerRedirecting()
2166
 
        redirecting.redirect_to(new.host, new.port)
2167
 
        return redirecting
2168
 
 
2169
2123
    def setUp(self):
2170
2124
        super(TestAuthOnRedirected, self).setUp()
2171
2125
        self.build_tree_contents([('a','a'),
2176
2130
                                       self.new_server.port)
2177
2131
        self.old_server.redirections = [
2178
2132
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2179
 
        self.old_transport = self._transport(self.old_server.get_url())
 
2133
        self.old_transport = self.get_old_transport()
2180
2134
        self.new_server.add_user('joe', 'foo')
2181
 
 
2182
 
    def get_a(self, transport):
2183
 
        return transport.get('a')
 
2135
        cleanup_http_redirection_connections(self)
 
2136
 
 
2137
    def create_transport_readonly_server(self):
 
2138
        server = self._auth_server(protocol_version=self._protocol_version)
 
2139
        server._url_protocol = self._url_protocol
 
2140
        return server
 
2141
 
 
2142
    def get_a(self, t):
 
2143
        return t.get('a')
2184
2144
 
2185
2145
    def test_auth_on_redirected_via_do_catching_redirections(self):
2186
2146
        self.redirections = 0
2187
2147
 
2188
 
        def redirected(transport, exception, redirection_notice):
 
2148
        def redirected(t, exception, redirection_notice):
2189
2149
            self.redirections += 1
2190
 
            dir, file = urlutils.split(exception.target)
2191
 
            return self._transport(dir)
 
2150
            redirected_t = t._redirected_to(exception.source, exception.target)
 
2151
            self.addCleanup(redirected_t.disconnect)
 
2152
            return redirected_t
2192
2153
 
2193
2154
        stdout = tests.StringIOWrapper()
2194
2155
        stderr = tests.StringIOWrapper()
2215
2176
                                       self.new_server.port)
2216
2177
        self.old_server.redirections = [
2217
2178
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2218
 
        self.assertEqual('redirected once',t._perform(req).read())
 
2179
        self.assertEqual('redirected once', t._perform(req).read())
2219
2180
        # stdin should be empty
2220
2181
        self.assertEqual('', ui.ui_factory.stdin.readline())
2221
2182
        # stdout should be empty, stderr will contains the prompts