1
# Copyright (C) 2006-2011 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Handlers for HTTP Responses.
19
The purpose of these classes is to provide a uniform interface for clients
20
to standard HTTP responses, single range responses and multipart range
25
from io import BytesIO
27
import http.client as http_client
28
import email.utils as email_utils
36
class ResponseFile(object):
37
"""A wrapper around the http socket containing the result of a GET request.
39
Only read() and seek() (forward) are supported.
43
def __init__(self, path, infile):
46
:param path: File url, for error reports.
48
:param infile: File-like socket set at body start.
57
Dummy implementation for consistency with the 'file' API.
63
def __exit__(self, exc_type, exc_val, exc_tb):
64
return False # propogate exceptions.
66
def read(self, size=None):
67
"""Read size bytes from the current position in the file.
69
:param size: The number of bytes to read. Leave unspecified or pass
72
data = self._file.read(size)
73
self._pos += len(data)
77
data = self._file.readline()
78
self._pos += len(data)
81
def readlines(self, size=None):
82
data = self._file.readlines()
83
self._pos += sum(map(len, data))
88
line = self.readline()
96
def seek(self, offset, whence=os.SEEK_SET):
97
if whence == os.SEEK_SET:
98
if offset < self._pos:
100
"Can't seek backwards, pos: %s, offset: %s"
101
% (self._pos, offset))
102
to_discard = offset - self._pos
103
elif whence == os.SEEK_CUR:
106
raise AssertionError("Can't seek backwards")
108
# Just discard the unwanted bytes
109
self.read(to_discard)
111
# A RangeFile expects the following grammar (simplified to outline the
112
# assumptions we rely upon).
117
# single_range: content_range_header data
119
# multiple_range: boundary_header boundary (content_range_header data boundary)+
122
class RangeFile(ResponseFile):
123
"""File-like object that allow access to partial available data.
125
All accesses should happen sequentially since the acquisition occurs during
126
an http response reception (as sockets can't be seeked, we simulate the
127
seek by just reading and discarding the data).
129
The access pattern is defined by a set of ranges discovered as reading
130
progress. Only one range is available at a given time, so all accesses
131
should happen with monotonically increasing offsets.
134
# in _checked_read() below, we may have to discard several MB in the worst
135
# case. To avoid buffering that much, we read and discard by chunks
136
# instead. The underlying file is either a socket or a BytesIO, so reading
137
# 8k chunks should be fine.
138
_discarded_buf_size = 8192
140
# maximum size of read requests -- used to avoid MemoryError issues in recv
141
_max_read_size = 512 * 1024
143
def __init__(self, path, infile):
146
:param path: File url, for error reports.
148
:param infile: File-like socket set at body start.
150
super(RangeFile, self).__init__(path, infile)
151
self._boundary = None
152
# When using multi parts response, this will be set with the headers
153
# associated with the range currently read.
155
# Default to the whole file of unspecified size
156
self.set_range(0, -1)
158
def set_range(self, start, size):
159
"""Change the range mapping"""
162
# Set the new _pos since that's what we want to expose
163
self._pos = self._start
165
def set_boundary(self, boundary):
166
"""Define the boundary used in a multi parts message.
168
The file should be at the beginning of the body, the first range
169
definition is read and taken into account.
171
if not isinstance(boundary, bytes):
172
raise TypeError(boundary)
173
self._boundary = boundary
174
# Decode the headers and setup the first range
176
self.read_range_definition()
178
def read_boundary(self):
179
"""Read the boundary headers defining a new range"""
180
boundary_line = b'\r\n'
181
while boundary_line == b'\r\n':
182
# RFC2616 19.2 Additional CRLFs may precede the first boundary
184
# To be on the safe side we allow it before any boundary line
185
boundary_line = self._file.readline()
187
if boundary_line == b'':
188
# A timeout in the proxy server caused the response to end early.
189
# See launchpad bug 198646.
190
raise errors.HttpBoundaryMissing(
194
if boundary_line != b'--' + self._boundary + b'\r\n':
195
# email_utils.unquote() incorrectly unquotes strings enclosed in <>
196
# IIS 6 and 7 incorrectly wrap boundary strings in <>
197
# together they make a beautiful bug, which we will be gracious
199
if (self._unquote_boundary(boundary_line) !=
200
b'--' + self._boundary + b'\r\n'):
201
raise errors.InvalidHttpResponse(
203
"Expected a boundary (%s) line, got '%s'"
204
% (self._boundary, boundary_line))
206
def _unquote_boundary(self, b):
207
return b[:2] + email_utils.unquote(b[2:-2].decode('ascii')).encode('ascii') + b[-2:]
209
def read_range_definition(self):
210
"""Read a new range definition in a multi parts message.
212
Parse the headers including the empty line following them so that we
213
are ready to read the data itself.
215
self._headers = http_client.parse_headers(self._file)
216
# Extract the range definition
217
content_range = self._headers.get('content-range', None)
218
if content_range is None:
219
raise errors.InvalidHttpResponse(
221
'Content-Range header missing in a multi-part response')
222
self.set_range_from_header(content_range)
224
def set_range_from_header(self, content_range):
225
"""Helper to set the new range from its description in the headers"""
227
rtype, values = content_range.split()
229
raise errors.InvalidHttpRange(self._path, content_range,
232
raise errors.InvalidHttpRange(self._path, content_range,
233
"Unsupported range type '%s'" % rtype)
235
# We don't need total, but note that it may be either the file size
236
# or '*' if the server can't or doesn't want to return the file
238
start_end, total = values.split('/')
239
start, end = start_end.split('-')
243
raise errors.InvalidHttpRange(self._path, content_range,
244
'Invalid range values')
245
size = end - start + 1
247
raise errors.InvalidHttpRange(self._path, content_range,
248
'Invalid range, size <= 0')
249
self.set_range(start, size)
251
def _checked_read(self, size):
252
"""Read the file checking for short reads.
254
The data read is discarded along the way.
259
data = self._file.read(min(remaining, self._discarded_buf_size))
260
remaining -= len(data)
262
raise errors.ShortReadvError(self._path, pos, size,
266
def _seek_to_next_range(self):
267
# We will cross range boundaries
268
if self._boundary is None:
269
# If we don't have a boundary, we can't find another range
270
raise errors.InvalidRange(self._path, self._pos,
271
"Range (%s, %s) exhausted"
272
% (self._start, self._size))
274
self.read_range_definition()
276
def read(self, size=-1):
277
"""Read size bytes from the current position in the file.
279
Reading across ranges is not supported. We rely on the underlying http
280
client to clean the socket if we leave bytes unread. This may occur for
281
the final boundary line of a multipart response or for any range
282
request not entirely consumed by the client (due to offset coalescing)
284
:param size: The number of bytes to read. Leave unspecified or pass
288
and self._pos == self._start + self._size):
292
self._seek_to_next_range()
293
elif self._pos < self._start:
294
raise errors.InvalidRange(
295
self._path, self._pos,
296
"Can't read %s bytes before range (%s, %s)"
297
% (size, self._start, self._size))
299
if size > 0 and self._pos + size > self._start + self._size:
300
raise errors.InvalidRange(
301
self._path, self._pos,
302
"Can't read %s bytes across range (%s, %s)"
303
% (size, self._start, self._size))
305
# read data from file
309
# Don't read past the range definition
310
limited = self._start + self._size - self._pos
312
limited = min(limited, size)
313
osutils.pumpfile(self._file, buf, limited, self._max_read_size)
314
data = buf.getvalue()
316
# Update _pos respecting the data effectively read
317
self._pos += len(data)
320
def seek(self, offset, whence=0):
321
start_pos = self._pos
325
final_pos = start_pos + offset
328
final_pos = self._start + self._size + offset # offset < 0
330
raise errors.InvalidRange(
331
self._path, self._pos,
332
"RangeFile: can't seek from end while size is unknown")
334
raise ValueError("Invalid value %s for whence." % whence)
336
if final_pos < self._pos:
337
# Can't seek backwards
338
raise errors.InvalidRange(
339
self._path, self._pos,
340
'RangeFile: trying to seek backwards to %s' % final_pos)
343
cur_limit = self._start + self._size
344
while final_pos > cur_limit:
345
# We will cross range boundaries
346
remain = cur_limit - self._pos
348
# Finish reading the current range
349
self._checked_read(remain)
350
self._seek_to_next_range()
351
cur_limit = self._start + self._size
353
size = final_pos - self._pos
354
if size > 0: # size can be < 0 if we crossed a range boundary
355
# We don't need the data, just read it and throw it away
356
self._checked_read(size)
362
def handle_response(url, code, getheader, data):
363
"""Interpret the code & headers and wrap the provided data in a RangeFile.
365
This is a factory method which returns an appropriate RangeFile based on
366
the code & headers it's given.
368
:param url: The url being processed. Mostly for error reporting
369
:param code: The integer HTTP response code
370
:param getheader: Function for retrieving header
371
:param data: A file-like object that can be read() to get the
373
:return: A file-like object that can seek()+read() the
374
ranges indicated by the headers.
378
rfile = ResponseFile(url, data)
380
rfile = RangeFile(url, data)
381
# When there is no content-type header we treat the response as
382
# being of type 'application/octet-stream' as per RFC2616 section
384
# Therefore it is obviously not multipart
385
content_type = getheader('content-type', 'application/octet-stream')
386
mimetype, options = cgi.parse_header(content_type)
387
if mimetype == 'multipart/byteranges':
388
rfile.set_boundary(options['boundary'].encode('ascii'))
390
# A response to a range request, but not multipart
391
content_range = getheader('content-range', None)
392
if content_range is None:
393
raise errors.InvalidHttpResponse(
394
url, 'Missing the Content-Range header in a 206 range response')
395
rfile.set_range_from_header(content_range)
397
raise errors.UnexpectedHttpStatus(url, code)
added
removed
breezy/transport/http/response.py
