1
# Copyright (C) 2006-2011 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Handlers for HTTP Responses.
19
The purpose of these classes is to provide a uniform interface for clients
20
to standard HTTP responses, single range responses and multipart range
25
from io import BytesIO
27
import http.client as http_client
28
import email.utils as email_utils
36
class ResponseFile(object):
37
"""A wrapper around the http socket containing the result of a GET request.
39
Only read() and seek() (forward) are supported.
43
def __init__(self, path, infile):
46
:param path: File url, for error reports.
48
:param infile: File-like socket set at body start.
57
Dummy implementation for consistency with the 'file' API.
63
def __exit__(self, exc_type, exc_val, exc_tb):
64
return False # propogate exceptions.
66
def read(self, size=None):
67
"""Read size bytes from the current position in the file.
69
:param size: The number of bytes to read. Leave unspecified or pass
72
data = self._file.read(size)
73
self._pos += len(data)
77
data = self._file.readline()
78
self._pos += len(data)
83
line = self.readline()
91
def seek(self, offset, whence=os.SEEK_SET):
92
if whence == os.SEEK_SET:
93
if offset < self._pos:
95
"Can't seek backwards, pos: %s, offset: %s"
96
% (self._pos, offset))
97
to_discard = offset - self._pos
98
elif whence == os.SEEK_CUR:
101
raise AssertionError("Can't seek backwards")
103
# Just discard the unwanted bytes
104
self.read(to_discard)
106
# A RangeFile expects the following grammar (simplified to outline the
107
# assumptions we rely upon).
112
# single_range: content_range_header data
114
# multiple_range: boundary_header boundary (content_range_header data boundary)+
117
class RangeFile(ResponseFile):
118
"""File-like object that allow access to partial available data.
120
All accesses should happen sequentially since the acquisition occurs during
121
an http response reception (as sockets can't be seeked, we simulate the
122
seek by just reading and discarding the data).
124
The access pattern is defined by a set of ranges discovered as reading
125
progress. Only one range is available at a given time, so all accesses
126
should happen with monotonically increasing offsets.
129
# in _checked_read() below, we may have to discard several MB in the worst
130
# case. To avoid buffering that much, we read and discard by chunks
131
# instead. The underlying file is either a socket or a BytesIO, so reading
132
# 8k chunks should be fine.
133
_discarded_buf_size = 8192
135
# maximum size of read requests -- used to avoid MemoryError issues in recv
136
_max_read_size = 512 * 1024
138
def __init__(self, path, infile):
141
:param path: File url, for error reports.
143
:param infile: File-like socket set at body start.
145
super(RangeFile, self).__init__(path, infile)
146
self._boundary = None
147
# When using multi parts response, this will be set with the headers
148
# associated with the range currently read.
150
# Default to the whole file of unspecified size
151
self.set_range(0, -1)
153
def set_range(self, start, size):
154
"""Change the range mapping"""
157
# Set the new _pos since that's what we want to expose
158
self._pos = self._start
160
def set_boundary(self, boundary):
161
"""Define the boundary used in a multi parts message.
163
The file should be at the beginning of the body, the first range
164
definition is read and taken into account.
166
if not isinstance(boundary, bytes):
167
raise TypeError(boundary)
168
self._boundary = boundary
169
# Decode the headers and setup the first range
171
self.read_range_definition()
173
def read_boundary(self):
174
"""Read the boundary headers defining a new range"""
175
boundary_line = b'\r\n'
176
while boundary_line == b'\r\n':
177
# RFC2616 19.2 Additional CRLFs may precede the first boundary
179
# To be on the safe side we allow it before any boundary line
180
boundary_line = self._file.readline()
182
if boundary_line == b'':
183
# A timeout in the proxy server caused the response to end early.
184
# See launchpad bug 198646.
185
raise errors.HttpBoundaryMissing(
189
if boundary_line != b'--' + self._boundary + b'\r\n':
190
# email_utils.unquote() incorrectly unquotes strings enclosed in <>
191
# IIS 6 and 7 incorrectly wrap boundary strings in <>
192
# together they make a beautiful bug, which we will be gracious
194
if (self._unquote_boundary(boundary_line) !=
195
b'--' + self._boundary + b'\r\n'):
196
raise errors.InvalidHttpResponse(
198
"Expected a boundary (%s) line, got '%s'"
199
% (self._boundary, boundary_line))
201
def _unquote_boundary(self, b):
202
return b[:2] + email_utils.unquote(b[2:-2].decode('ascii')).encode('ascii') + b[-2:]
204
def read_range_definition(self):
205
"""Read a new range definition in a multi parts message.
207
Parse the headers including the empty line following them so that we
208
are ready to read the data itself.
210
self._headers = http_client.parse_headers(self._file)
211
# Extract the range definition
212
content_range = self._headers.get('content-range', None)
213
if content_range is None:
214
raise errors.InvalidHttpResponse(
216
'Content-Range header missing in a multi-part response')
217
self.set_range_from_header(content_range)
219
def set_range_from_header(self, content_range):
220
"""Helper to set the new range from its description in the headers"""
222
rtype, values = content_range.split()
224
raise errors.InvalidHttpRange(self._path, content_range,
227
raise errors.InvalidHttpRange(self._path, content_range,
228
"Unsupported range type '%s'" % rtype)
230
# We don't need total, but note that it may be either the file size
231
# or '*' if the server can't or doesn't want to return the file
233
start_end, total = values.split('/')
234
start, end = start_end.split('-')
238
raise errors.InvalidHttpRange(self._path, content_range,
239
'Invalid range values')
240
size = end - start + 1
242
raise errors.InvalidHttpRange(self._path, content_range,
243
'Invalid range, size <= 0')
244
self.set_range(start, size)
246
def _checked_read(self, size):
247
"""Read the file checking for short reads.
249
The data read is discarded along the way.
254
data = self._file.read(min(remaining, self._discarded_buf_size))
255
remaining -= len(data)
257
raise errors.ShortReadvError(self._path, pos, size,
261
def _seek_to_next_range(self):
262
# We will cross range boundaries
263
if self._boundary is None:
264
# If we don't have a boundary, we can't find another range
265
raise errors.InvalidRange(self._path, self._pos,
266
"Range (%s, %s) exhausted"
267
% (self._start, self._size))
269
self.read_range_definition()
271
def read(self, size=-1):
272
"""Read size bytes from the current position in the file.
274
Reading across ranges is not supported. We rely on the underlying http
275
client to clean the socket if we leave bytes unread. This may occur for
276
the final boundary line of a multipart response or for any range
277
request not entirely consumed by the client (due to offset coalescing)
279
:param size: The number of bytes to read. Leave unspecified or pass
283
and self._pos == self._start + self._size):
287
self._seek_to_next_range()
288
elif self._pos < self._start:
289
raise errors.InvalidRange(
290
self._path, self._pos,
291
"Can't read %s bytes before range (%s, %s)"
292
% (size, self._start, self._size))
294
if size > 0 and self._pos + size > self._start + self._size:
295
raise errors.InvalidRange(
296
self._path, self._pos,
297
"Can't read %s bytes across range (%s, %s)"
298
% (size, self._start, self._size))
300
# read data from file
304
# Don't read past the range definition
305
limited = self._start + self._size - self._pos
307
limited = min(limited, size)
308
osutils.pumpfile(self._file, buf, limited, self._max_read_size)
309
data = buf.getvalue()
311
# Update _pos respecting the data effectively read
312
self._pos += len(data)
315
def seek(self, offset, whence=0):
316
start_pos = self._pos
320
final_pos = start_pos + offset
323
final_pos = self._start + self._size + offset # offset < 0
325
raise errors.InvalidRange(
326
self._path, self._pos,
327
"RangeFile: can't seek from end while size is unknown")
329
raise ValueError("Invalid value %s for whence." % whence)
331
if final_pos < self._pos:
332
# Can't seek backwards
333
raise errors.InvalidRange(
334
self._path, self._pos,
335
'RangeFile: trying to seek backwards to %s' % final_pos)
338
cur_limit = self._start + self._size
339
while final_pos > cur_limit:
340
# We will cross range boundaries
341
remain = cur_limit - self._pos
343
# Finish reading the current range
344
self._checked_read(remain)
345
self._seek_to_next_range()
346
cur_limit = self._start + self._size
348
size = final_pos - self._pos
349
if size > 0: # size can be < 0 if we crossed a range boundary
350
# We don't need the data, just read it and throw it away
351
self._checked_read(size)
357
def handle_response(url, code, getheader, data):
358
"""Interpret the code & headers and wrap the provided data in a RangeFile.
360
This is a factory method which returns an appropriate RangeFile based on
361
the code & headers it's given.
363
:param url: The url being processed. Mostly for error reporting
364
:param code: The integer HTTP response code
365
:param getheader: Function for retrieving header
366
:param data: A file-like object that can be read() to get the
368
:return: A file-like object that can seek()+read() the
369
ranges indicated by the headers.
373
rfile = ResponseFile(url, data)
375
rfile = RangeFile(url, data)
376
# When there is no content-type header we treat the response as
377
# being of type 'application/octet-stream' as per RFC2616 section
379
# Therefore it is obviously not multipart
380
content_type = getheader('content-type', 'application/octet-stream')
381
mimetype, options = cgi.parse_header(content_type)
382
if mimetype == 'multipart/byteranges':
383
rfile.set_boundary(options['boundary'].encode('ascii'))
385
# A response to a range request, but not multipart
386
content_range = getheader('content-range', None)
387
if content_range is None:
388
raise errors.InvalidHttpResponse(
389
url, 'Missing the Content-Range header in a 206 range response')
390
rfile.set_range_from_header(content_range)
392
raise errors.InvalidHttpResponse(url,
393
'Unknown response code %s' % code)
added
removed
breezy/transport/http/response.py
