66
65
return self.readfile
69
class FakeHTTPConnection(HTTPConnection):
68
class FakeHTTPConnection(_urllib2_wrappers.HTTPConnection):
71
70
def __init__(self, sock):
72
HTTPConnection.__init__(self, 'localhost')
71
_urllib2_wrappers.HTTPConnection.__init__(self, 'localhost')
73
72
# Set the socket to bypass the connection
87
86
def test_iter_many(self):
88
87
f = response.ResponseFile('many', BytesIO(b'0\n1\nboo!\n'))
89
self.assertEqual([b'0\n', b'1\n', b'boo!\n'], list(f))
91
def test_readlines(self):
92
f = response.ResponseFile('many', BytesIO(b'0\n1\nboo!\n'))
93
self.assertEqual([b'0\n', b'1\n', b'boo!\n'], f.readlines())
88
self.assertEqual(['0\n', '1\n', 'boo!\n'], list(f))
96
91
class TestHTTPConnection(tests.TestCase):
98
93
def test_cleanup_pipe(self):
99
sock = ReadSocket(b"""HTTP/1.1 200 OK\r
94
sock = ReadSocket("""HTTP/1.1 200 OK\r
100
95
Content-Type: text/plain; charset=UTF-8\r
101
96
Content-Length: 18
110
105
# Now, get the response
111
106
resp = conn.getresponse()
112
107
# Read part of the response
113
self.assertEqual(b'0123456789\n', resp.read(11))
108
self.assertEqual('0123456789\n', resp.read(11))
114
109
# Override the thresold to force the warning emission
115
conn._range_warning_thresold = 6 # There are 7 bytes pending
110
conn._range_warning_thresold = 6 # There are 7 bytes pending
116
111
conn.cleanup_pipe()
117
112
self.assertContainsRe(self.get_log(), 'Got a 200 response when asking')
124
119
# which offsets are easy to calculate for test writers. It's used as a
125
120
# building block with slight variations but basically 'a' is the first char
126
121
# of the range and 'z' is the last.
127
alpha = b'abcdefghijklmnopqrstuvwxyz'
122
alpha = 'abcdefghijklmnopqrstuvwxyz'
129
124
def test_can_read_at_first_access(self):
130
125
"""Test that the just created file can be read."""
136
131
start = self.first_range_start
137
132
# Before any use, tell() should be at the range start
138
133
self.assertEqual(start, f.tell())
139
cur = start # For an overall offset assertion
134
cur = start # For an overall offset assertion
140
135
f.seek(start + 3)
142
self.assertEqual(b'def', f.read(3))
137
self.assertEqual('def', f.read(3))
143
138
cur += len('def')
146
self.assertEqual(b'klmn', f.read(4))
141
self.assertEqual('klmn', f.read(4))
147
142
cur += len('klmn')
148
143
# read(0) in the middle of a range
149
self.assertEqual(b'', f.read(0))
144
self.assertEqual('', f.read(0))
156
151
def test_read_zero(self):
158
self.assertEqual(b'', f.read(0))
153
self.assertEqual('', f.read(0))
160
self.assertEqual(b'', f.read(0))
155
self.assertEqual('', f.read(0))
162
157
def test_seek_at_range_end(self):
167
162
"""Test read behaviour at range end."""
169
164
self.assertEqual(self.alpha, f.read())
170
self.assertEqual(b'', f.read(0))
165
self.assertEqual('', f.read(0))
171
166
self.assertRaises(errors.InvalidRange, f.read, 1)
173
168
def test_unbounded_read_after_seek(self):
176
171
# Should not cross ranges
177
self.assertEqual(b'yz', f.read())
172
self.assertEqual('yz', f.read())
179
174
def test_seek_backwards(self):
200
195
self.assertRaises(errors.InvalidRange, f.read, 10)
202
197
def test_seek_from_end(self):
203
"""Test seeking from the end of the file.
205
The semantic is unclear in case of multiple ranges. Seeking from end
206
exists only for the http transports, cannot be used if the file size is
207
unknown and is not used in breezy itself. This test must be (and is)
208
overridden by daughter classes.
210
Reading from end makes sense only when a range has been requested from
211
the end of the file (see HttpTransportBase._get() when using the
212
'tail_amount' parameter). The HTTP response can only be a whole file or
217
self.assertEqual(b'yz', f.read())
198
"""Test seeking from the end of the file.
200
The semantic is unclear in case of multiple ranges. Seeking from end
201
exists only for the http transports, cannot be used if the file size is
202
unknown and is not used in breezy itself. This test must be (and is)
203
overridden by daughter classes.
205
Reading from end makes sense only when a range has been requested from
206
the end of the file (see HttpTransportBase._get() when using the
207
'tail_amount' parameter). The HTTP response can only be a whole file or
212
self.assertEqual('yz', f.read())
220
215
class TestRangeFileSizeUnknown(tests.TestCase, TestRangeFileMixin):
225
220
self._file = response.RangeFile('Whole_file_size_known',
226
221
BytesIO(self.alpha))
227
222
# We define no range, relying on RangeFile to provide default values
228
self.first_range_start = 0 # It's the whole file
223
self.first_range_start = 0 # It's the whole file
230
225
def test_seek_from_end(self):
231
226
"""See TestRangeFileMixin.test_seek_from_end.
238
233
"""Test read behaviour at range end."""
240
235
self.assertEqual(self.alpha, f.read())
241
self.assertEqual(b'', f.read(0))
242
self.assertEqual(b'', f.read(1))
236
self.assertEqual('', f.read(0))
237
self.assertEqual('', f.read(1))
245
240
class TestRangeFileSizeKnown(tests.TestCase, TestRangeFileMixin):
263
258
self.first_range_start = 15
264
259
self._file.set_range(self.first_range_start, len(self.alpha))
266
262
def test_read_before_range(self):
267
263
# This can't occur under normal circumstances, we have to force it
269
f._pos = 0 # Force an invalid pos
265
f._pos = 0 # Force an invalid pos
270
266
self.assertRaises(errors.InvalidRange, f.read, 2)
287
283
# in HTTP response headers and the boundary lines that separate
288
284
# multipart content.
290
boundary = b"separation"
286
boundary = "separation"
293
289
super(TestRangeFileMultipleRanges, self).setUp()
295
291
boundary = self.boundary
298
294
self.first_range_start = 25
299
file_size = 200 # big enough to encompass all ranges
295
file_size = 200 # big enough to encompass all ranges
300
296
for (start, part) in [(self.first_range_start, self.alpha),
301
297
# Two contiguous ranges
302
298
(100, self.alpha),
321
317
# which is part of the Content-Type header).
322
318
self._file.set_boundary(self.boundary)
324
def _multipart_byterange(self, data, offset, boundary, file_size=b'*'):
320
def _multipart_byterange(self, data, offset, boundary, file_size='*'):
325
321
"""Encode a part of a file as a multipart/byterange MIME type.
327
323
When a range request is issued, the HTTP response body can be
343
339
# A range is described by a set of headers, but only 'Content-Range' is
344
340
# required for our implementation (TestHandleResponse below will
345
341
# exercise ranges with multiple or missing headers')
346
if isinstance(file_size, int):
347
file_size = b'%d' % file_size
348
range += b'Content-Range: bytes %d-%d/%s\r\n' % (offset,
342
range += 'Content-Range: bytes %d-%d/%d\r\n' % (offset,
353
346
# Finally the raw bytes
357
350
def test_read_all_ranges(self):
359
self.assertEqual(self.alpha, f.read()) # Read first range
360
f.seek(100) # Trigger the second range recognition
361
self.assertEqual(self.alpha, f.read()) # Read second range
352
self.assertEqual(self.alpha, f.read()) # Read first range
353
f.seek(100) # Trigger the second range recognition
354
self.assertEqual(self.alpha, f.read()) # Read second range
362
355
self.assertEqual(126, f.tell())
363
f.seek(126) # Start of third range which is also the current pos !
364
self.assertEqual(b'A', f.read(1))
356
f.seek(126) # Start of third range which is also the current pos !
357
self.assertEqual('A', f.read(1))
366
self.assertEqual(b'LMN', f.read(3))
359
self.assertEqual('LMN', f.read(3))
368
361
def test_seek_from_end(self):
369
362
"""See TestRangeFileMixin.test_seek_from_end."""
391
384
def test_seek_across_ranges(self):
393
f.seek(126) # skip the two first ranges
394
self.assertEqual(b'AB', f.read(2))
386
f.seek(126) # skip the two first ranges
387
self.assertEqual('AB', f.read(2))
396
389
def test_checked_read_dont_overflow_buffers(self):
398
391
# We force a very low value to exercise all code paths in _checked_read
399
392
f._discarded_buf_size = 8
400
f.seek(126) # skip the two first ranges
401
self.assertEqual(b'AB', f.read(2))
393
f.seek(126) # skip the two first ranges
394
self.assertEqual('AB', f.read(2))
403
396
def test_seek_twice_between_ranges(self):
405
398
start = self.first_range_start
406
f.seek(start + 40) # Past the first range but before the second
399
f.seek(start + 40) # Past the first range but before the second
407
400
# Now the file is positioned at the second range start (100)
408
401
self.assertRaises(errors.InvalidRange, f.seek, start + 41)
440
433
# The boundary as it appears in boundary lines
441
434
# IIS 6 and 7 use this value
442
_boundary_trimmed = b"q1w2e3r4t5y6u7i8o9p0zaxscdvfbgnhmjklkl"
443
boundary = b'<' + _boundary_trimmed + b'>'
435
_boundary_trimmed = "q1w2e3r4t5y6u7i8o9p0zaxscdvfbgnhmjklkl"
436
boundary = '<' + _boundary_trimmed + '>'
445
438
def set_file_boundary(self):
446
439
# Emulate broken rfc822.unquote() here by removing angles
474
467
ok((12, 2), '\tbytes 12-13/*')
475
468
ok((28, 1), ' bytes 28-28/*')
476
469
ok((2123, 2120), 'bytes 2123-4242/12310')
477
ok((1, 10), 'bytes 1-10/ttt') # We don't check total (ttt)
470
ok((1, 10), 'bytes 1-10/ttt') # We don't check total (ttt)
479
472
def nok(header_value):
480
473
self.assertRaises(errors.InvalidHttpRange,
491
484
# Taken from real request responses
492
_full_text_response = (200, b"""HTTP/1.1 200 OK\r
485
_full_text_response = (200, """HTTP/1.1 200 OK\r
493
486
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
494
487
Server: Apache/2.0.54 (Fedora)\r
495
488
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
499
492
Connection: close\r
500
493
Content-Type: text/plain; charset=UTF-8\r
502
""", b"""Bazaar-NG meta directory, format 1
495
""", """Bazaar-NG meta directory, format 1
506
_single_range_response = (206, b"""HTTP/1.1 206 Partial Content\r
499
_single_range_response = (206, """HTTP/1.1 206 Partial Content\r
507
500
Date: Tue, 11 Jul 2006 04:45:22 GMT\r
508
501
Server: Apache/2.0.54 (Fedora)\r
509
502
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
514
507
Connection: close\r
515
508
Content-Type: text/plain; charset=UTF-8\r
517
""", b"""mbp@sourcefrog.net-20050309040815-13242001617e4a06
510
""", """mbp@sourcefrog.net-20050309040815-13242001617e4a06
518
511
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e762""")
521
_single_range_no_content_type = (206, b"""HTTP/1.1 206 Partial Content\r
514
_single_range_no_content_type = (206, """HTTP/1.1 206 Partial Content\r
522
515
Date: Tue, 11 Jul 2006 04:45:22 GMT\r
523
516
Server: Apache/2.0.54 (Fedora)\r
524
517
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
528
521
Content-Range: bytes 100-199/93890\r
529
522
Connection: close\r
531
""", b"""mbp@sourcefrog.net-20050309040815-13242001617e4a06
524
""", """mbp@sourcefrog.net-20050309040815-13242001617e4a06
532
525
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e762""")
535
_multipart_range_response = (206, b"""HTTP/1.1 206 Partial Content\r
528
_multipart_range_response = (206, """HTTP/1.1 206 Partial Content\r
536
529
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
537
530
Server: Apache/2.0.54 (Fedora)\r
538
531
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
624
617
Content-Length: 35\r
625
618
Connection: close\r
627
""", b"""Bazaar-NG meta directory, format 1
620
""", """Bazaar-NG meta directory, format 1
631
_full_text_response_no_content_length = (200, b"""HTTP/1.1 200 OK\r
624
_full_text_response_no_content_length = (200, """HTTP/1.1 200 OK\r
632
625
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
633
626
Server: Apache/2.0.54 (Fedora)\r
634
627
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
637
630
Connection: close\r
638
631
Content-Type: text/plain; charset=UTF-8\r
640
""", b"""Bazaar-NG meta directory, format 1
633
""", """Bazaar-NG meta directory, format 1
644
_single_range_no_content_range = (206, b"""HTTP/1.1 206 Partial Content\r
637
_single_range_no_content_range = (206, """HTTP/1.1 206 Partial Content\r
645
638
Date: Tue, 11 Jul 2006 04:45:22 GMT\r
646
639
Server: Apache/2.0.54 (Fedora)\r
647
640
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
650
643
Content-Length: 100\r
651
644
Connection: close\r
653
""", b"""mbp@sourcefrog.net-20050309040815-13242001617e4a06
646
""", """mbp@sourcefrog.net-20050309040815-13242001617e4a06
654
647
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e762""")
657
_single_range_response_truncated = (206, b"""HTTP/1.1 206 Partial Content\r
650
_single_range_response_truncated = (206, """HTTP/1.1 206 Partial Content\r
658
651
Date: Tue, 11 Jul 2006 04:45:22 GMT\r
659
652
Server: Apache/2.0.54 (Fedora)\r
660
653
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
665
658
Connection: close\r
666
659
Content-Type: text/plain; charset=UTF-8\r
668
""", b"""mbp@sourcefrog.net-20050309040815-13242001617e4a06""")
671
_invalid_response = (444, b"""HTTP/1.1 444 Bad Response\r
661
""", """mbp@sourcefrog.net-20050309040815-13242001617e4a06""")
664
_invalid_response = (444, """HTTP/1.1 444 Bad Response\r
672
665
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
673
666
Connection: close\r
674
667
Content-Type: text/html; charset=iso-8859-1\r
676
""", b"""<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
669
""", """<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
678
671
<title>404 Not Found</title>
721
714
status_and_headers = BytesIO(raw_headers)
722
715
# Get rid of the status line
723
716
status_and_headers.readline()
724
msg = parse_headers(status_and_headers)
717
msg = httplib.HTTPMessage(status_and_headers)
727
720
def get_response(self, a_response):
728
721
"""Process a supplied response, and return the result."""
729
722
code, raw_headers, body = a_response
730
getheader = self._build_HTTPMessage(raw_headers)
731
return response.handle_response(
732
'http://foo', code, getheader, BytesIO(a_response[2]))
723
msg = self._build_HTTPMessage(raw_headers)
724
return response.handle_response('http://foo', code, msg,
725
BytesIO(a_response[2]))
734
727
def test_full_text(self):
735
728
out = self.get_response(_full_text_response)
780
773
def test_full_text_no_content_type(self):
781
774
# We should not require Content-Type for a full response
782
775
code, raw_headers, body = _full_text_response_no_content_type
783
getheader = self._build_HTTPMessage(raw_headers)
784
out = response.handle_response(
785
'http://foo', code, getheader, BytesIO(body))
776
msg = self._build_HTTPMessage(raw_headers)
777
out = response.handle_response('http://foo', code, msg, BytesIO(body))
786
778
self.assertEqual(body, out.read())
788
780
def test_full_text_no_content_length(self):
789
781
code, raw_headers, body = _full_text_response_no_content_length
790
getheader = self._build_HTTPMessage(raw_headers)
791
out = response.handle_response(
792
'http://foo', code, getheader, BytesIO(body))
782
msg = self._build_HTTPMessage(raw_headers)
783
out = response.handle_response('http://foo', code, msg, BytesIO(body))
793
784
self.assertEqual(body, out.read())
795
786
def test_missing_content_range(self):
796
787
code, raw_headers, body = _single_range_no_content_range
797
getheader = self._build_HTTPMessage(raw_headers)
788
msg = self._build_HTTPMessage(raw_headers)
798
789
self.assertRaises(errors.InvalidHttpResponse,
799
790
response.handle_response,
800
'http://bogus', code, getheader, BytesIO(body))
791
'http://bogus', code, msg, BytesIO(body))
802
793
def test_multipart_no_content_range(self):
803
794
code, raw_headers, body = _multipart_no_content_range
804
getheader = self._build_HTTPMessage(raw_headers)
795
msg = self._build_HTTPMessage(raw_headers)
805
796
self.assertRaises(errors.InvalidHttpResponse,
806
797
response.handle_response,
807
'http://bogus', code, getheader, BytesIO(body))
798
'http://bogus', code, msg, BytesIO(body))
809
800
def test_multipart_no_boundary(self):
810
801
out = self.get_response(_multipart_no_boundary)
822
813
super(TestRangeFileSizeReadLimited, self).setUp()
823
814
# create a test datablock larger than _max_read_size.
824
815
chunk_size = response.RangeFile._max_read_size
825
test_pattern = b'0123456789ABCDEF'
826
self.test_data = test_pattern * (3 * chunk_size // len(test_pattern))
816
test_pattern = '0123456789ABCDEF'
817
self.test_data = test_pattern * (3 * chunk_size / len(test_pattern))
827
818
self.test_data_len = len(self.test_data)
829
820
def test_max_read_size(self):