1
# Copyright (C) 2006-2011 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Handlers for HTTP Responses.
19
The purpose of these classes is to provide a uniform interface for clients
20
to standard HTTP responses, single range responses and multipart range
24
from __future__ import absolute_import
29
import http.client as http_client
30
except ImportError: # python < 3
31
import httplib as http_client
33
import email.utils as email_utils
34
except ImportError: # python < 3
35
import rfc822 as email_utils
41
from ...sixish import (
47
class ResponseFile(object):
48
"""A wrapper around the http socket containing the result of a GET request.
50
Only read() and seek() (forward) are supported.
54
def __init__(self, path, infile):
57
:param path: File url, for error reports.
59
:param infile: File-like socket set at body start.
68
Dummy implementation for consistency with the 'file' API.
74
def __exit__(self, exc_type, exc_val, exc_tb):
75
return False # propogate exceptions.
77
def read(self, size=None):
78
"""Read size bytes from the current position in the file.
80
:param size: The number of bytes to read. Leave unspecified or pass
83
if size is None and not PY3:
85
data = self._file.read(size)
86
self._pos += len(data)
90
data = self._file.readline()
91
self._pos += len(data)
96
line = self.readline()
104
def seek(self, offset, whence=os.SEEK_SET):
105
if whence == os.SEEK_SET:
106
if offset < self._pos:
107
raise AssertionError(
108
"Can't seek backwards, pos: %s, offset: %s"
109
% (self._pos, offset))
110
to_discard = offset - self._pos
111
elif whence == os.SEEK_CUR:
114
raise AssertionError("Can't seek backwards")
116
# Just discard the unwanted bytes
117
self.read(to_discard)
119
# A RangeFile expects the following grammar (simplified to outline the
120
# assumptions we rely upon).
125
# single_range: content_range_header data
127
# multiple_range: boundary_header boundary (content_range_header data boundary)+
130
class RangeFile(ResponseFile):
131
"""File-like object that allow access to partial available data.
133
All accesses should happen sequentially since the acquisition occurs during
134
an http response reception (as sockets can't be seeked, we simulate the
135
seek by just reading and discarding the data).
137
The access pattern is defined by a set of ranges discovered as reading
138
progress. Only one range is available at a given time, so all accesses
139
should happen with monotonically increasing offsets.
142
# in _checked_read() below, we may have to discard several MB in the worst
143
# case. To avoid buffering that much, we read and discard by chunks
144
# instead. The underlying file is either a socket or a BytesIO, so reading
145
# 8k chunks should be fine.
146
_discarded_buf_size = 8192
148
# maximum size of read requests -- used to avoid MemoryError issues in recv
149
_max_read_size = 512 * 1024
151
def __init__(self, path, infile):
154
:param path: File url, for error reports.
156
:param infile: File-like socket set at body start.
158
super(RangeFile, self).__init__(path, infile)
159
self._boundary = None
160
# When using multi parts response, this will be set with the headers
161
# associated with the range currently read.
163
# Default to the whole file of unspecified size
164
self.set_range(0, -1)
166
def set_range(self, start, size):
167
"""Change the range mapping"""
170
# Set the new _pos since that's what we want to expose
171
self._pos = self._start
173
def set_boundary(self, boundary):
174
"""Define the boundary used in a multi parts message.
176
The file should be at the beginning of the body, the first range
177
definition is read and taken into account.
179
if not isinstance(boundary, bytes):
180
raise TypeError(boundary)
181
self._boundary = boundary
182
# Decode the headers and setup the first range
184
self.read_range_definition()
186
def read_boundary(self):
187
"""Read the boundary headers defining a new range"""
188
boundary_line = b'\r\n'
189
while boundary_line == b'\r\n':
190
# RFC2616 19.2 Additional CRLFs may precede the first boundary
192
# To be on the safe side we allow it before any boundary line
193
boundary_line = self._file.readline()
195
if boundary_line == b'':
196
# A timeout in the proxy server caused the response to end early.
197
# See launchpad bug 198646.
198
raise errors.HttpBoundaryMissing(
202
if boundary_line != b'--' + self._boundary + b'\r\n':
203
# email_utils.unquote() incorrectly unquotes strings enclosed in <>
204
# IIS 6 and 7 incorrectly wrap boundary strings in <>
205
# together they make a beautiful bug, which we will be gracious
207
if (self._unquote_boundary(boundary_line) !=
208
b'--' + self._boundary + b'\r\n'):
209
raise errors.InvalidHttpResponse(
211
"Expected a boundary (%s) line, got '%s'"
212
% (self._boundary, boundary_line))
214
def _unquote_boundary(self, b):
215
return b[:2] + email_utils.unquote(b[2:-2].decode('ascii')).encode('ascii') + b[-2:]
217
def read_range_definition(self):
218
"""Read a new range definition in a multi parts message.
220
Parse the headers including the empty line following them so that we
221
are ready to read the data itself.
224
self._headers = http_client.parse_headers(self._file)
226
self._headers = http_client.HTTPMessage(self._file, seekable=0)
227
# Extract the range definition
228
content_range = self._headers.get('content-range', None)
229
if content_range is None:
230
raise errors.InvalidHttpResponse(
232
'Content-Range header missing in a multi-part response')
233
self.set_range_from_header(content_range)
235
def set_range_from_header(self, content_range):
236
"""Helper to set the new range from its description in the headers"""
238
rtype, values = content_range.split()
240
raise errors.InvalidHttpRange(self._path, content_range,
243
raise errors.InvalidHttpRange(self._path, content_range,
244
"Unsupported range type '%s'" % rtype)
246
# We don't need total, but note that it may be either the file size
247
# or '*' if the server can't or doesn't want to return the file
249
start_end, total = values.split('/')
250
start, end = start_end.split('-')
254
raise errors.InvalidHttpRange(self._path, content_range,
255
'Invalid range values')
256
size = end - start + 1
258
raise errors.InvalidHttpRange(self._path, content_range,
259
'Invalid range, size <= 0')
260
self.set_range(start, size)
262
def _checked_read(self, size):
263
"""Read the file checking for short reads.
265
The data read is discarded along the way.
270
data = self._file.read(min(remaining, self._discarded_buf_size))
271
remaining -= len(data)
273
raise errors.ShortReadvError(self._path, pos, size,
277
def _seek_to_next_range(self):
278
# We will cross range boundaries
279
if self._boundary is None:
280
# If we don't have a boundary, we can't find another range
281
raise errors.InvalidRange(self._path, self._pos,
282
"Range (%s, %s) exhausted"
283
% (self._start, self._size))
285
self.read_range_definition()
287
def read(self, size=-1):
288
"""Read size bytes from the current position in the file.
290
Reading across ranges is not supported. We rely on the underlying http
291
client to clean the socket if we leave bytes unread. This may occur for
292
the final boundary line of a multipart response or for any range
293
request not entirely consumed by the client (due to offset coalescing)
295
:param size: The number of bytes to read. Leave unspecified or pass
299
and self._pos == self._start + self._size):
303
self._seek_to_next_range()
304
elif self._pos < self._start:
305
raise errors.InvalidRange(
306
self._path, self._pos,
307
"Can't read %s bytes before range (%s, %s)"
308
% (size, self._start, self._size))
310
if size > 0 and self._pos + size > self._start + self._size:
311
raise errors.InvalidRange(
312
self._path, self._pos,
313
"Can't read %s bytes across range (%s, %s)"
314
% (size, self._start, self._size))
316
# read data from file
320
# Don't read past the range definition
321
limited = self._start + self._size - self._pos
323
limited = min(limited, size)
324
osutils.pumpfile(self._file, buf, limited, self._max_read_size)
325
data = buf.getvalue()
327
# Update _pos respecting the data effectively read
328
self._pos += len(data)
331
def seek(self, offset, whence=0):
332
start_pos = self._pos
336
final_pos = start_pos + offset
339
final_pos = self._start + self._size + offset # offset < 0
341
raise errors.InvalidRange(
342
self._path, self._pos,
343
"RangeFile: can't seek from end while size is unknown")
345
raise ValueError("Invalid value %s for whence." % whence)
347
if final_pos < self._pos:
348
# Can't seek backwards
349
raise errors.InvalidRange(
350
self._path, self._pos,
351
'RangeFile: trying to seek backwards to %s' % final_pos)
354
cur_limit = self._start + self._size
355
while final_pos > cur_limit:
356
# We will cross range boundaries
357
remain = cur_limit - self._pos
359
# Finish reading the current range
360
self._checked_read(remain)
361
self._seek_to_next_range()
362
cur_limit = self._start + self._size
364
size = final_pos - self._pos
365
if size > 0: # size can be < 0 if we crossed a range boundary
366
# We don't need the data, just read it and throw it away
367
self._checked_read(size)
373
def handle_response(url, code, getheader, data):
374
"""Interpret the code & headers and wrap the provided data in a RangeFile.
376
This is a factory method which returns an appropriate RangeFile based on
377
the code & headers it's given.
379
:param url: The url being processed. Mostly for error reporting
380
:param code: The integer HTTP response code
381
:param getheader: Function for retrieving header
382
:param data: A file-like object that can be read() to get the
384
:return: A file-like object that can seek()+read() the
385
ranges indicated by the headers.
389
rfile = ResponseFile(url, data)
391
rfile = RangeFile(url, data)
392
# When there is no content-type header we treat the response as
393
# being of type 'application/octet-stream' as per RFC2616 section
395
# Therefore it is obviously not multipart
396
content_type = getheader('content-type', 'application/octet-stream')
397
mimetype, options = cgi.parse_header(content_type)
398
if mimetype == 'multipart/byteranges':
399
rfile.set_boundary(options['boundary'].encode('ascii'))
401
# A response to a range request, but not multipart
402
content_range = getheader('content-range', None)
403
if content_range is None:
404
raise errors.InvalidHttpResponse(
405
url, 'Missing the Content-Range header in a 206 range response')
406
rfile.set_range_from_header(content_range)
408
raise errors.InvalidHttpResponse(url,
409
'Unknown response code %s' % code)
added
removed
breezy/transport/http/response.py
