1
# Copyright (C) 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Handlers for HTTP Responses.
19
The purpose of these classes is to provide a uniform interface for clients
20
to standard HTTP responses, single range responses and multipart range
26
from cStringIO import StringIO
36
# A RangeFile expects the following grammar (simplified to outline the
37
# assumptions we rely upon).
43
# whole_file: [content_length_header] data
45
# single_range: content_range_header data
47
# multiple_range: boundary_header boundary (content_range_header data boundary)+
49
class RangeFile(object):
50
"""File-like object that allow access to partial available data.
52
All accesses should happen sequentially since the acquisition occurs during
53
an http response reception (as sockets can't be seeked, we simulate the
54
seek by just reading and discarding the data).
56
The access pattern is defined by a set of ranges discovered as reading
57
progress. Only one range is available at a given time, so all accesses
58
should happen with monotonically increasing offsets.
61
# in _checked_read() below, we may have to discard several MB in the worst
62
# case. To avoid buffering that much, we read and discard by chunks
63
# instead. The underlying file is either a socket or a StringIO, so reading
64
# 8k chunks should be fine.
65
_discarded_buf_size = 8192
67
# maximum size of read requests -- used to avoid MemoryError issues in recv
68
_max_read_size = 512 * 1024
70
def __init__(self, path, infile, report_activity=None):
73
:param path: File url, for error reports.
74
:param infile: File-like socket set at body start.
75
:param report_activity: A Transport._report_activity function to call
81
self._report_activity = report_activity
82
# When using multi parts response, this will be set with the headers
83
# associated with the range currently read.
85
# Default to the whole file of unspecified size
88
def set_range(self, start, size):
89
"""Change the range mapping"""
92
# Set the new _pos since that's what we want to expose
93
self._pos = self._start
95
def set_boundary(self, boundary):
96
"""Define the boundary used in a multi parts message.
98
The file should be at the beginning of the body, the first range
99
definition is read and taken into account.
101
self._boundary = boundary
102
# Decode the headers and setup the first range
104
self.read_range_definition()
106
def read_boundary(self):
107
"""Read the boundary headers defining a new range"""
108
boundary_line = '\r\n'
109
while boundary_line == '\r\n':
110
# RFC2616 19.2 Additional CRLFs may precede the first boundary
112
# To be on the safe side we allow it before any boundary line
113
boundary_line = self._file.readline()
115
if boundary_line != '--' + self._boundary + '\r\n':
116
# rfc822.unquote() incorrectly unquotes strings enclosed in <>
117
# IIS 6 and 7 incorrectly wrap boundary strings in <>
118
# together they make a beautiful bug, which we will be gracious
120
if (self._unquote_boundary(boundary_line) !=
121
'--' + self._boundary + '\r\n'):
122
raise errors.InvalidHttpResponse(
124
"Expected a boundary (%s) line, got '%s'"
125
% (self._boundary, boundary_line))
127
def _unquote_boundary(self, b):
128
return b[:2] + rfc822.unquote(b[2:-2]) + b[-2:]
130
def read_range_definition(self):
131
"""Read a new range definition in a multi parts message.
133
Parse the headers including the empty line following them so that we
134
are ready to read the data itself.
136
self._headers = httplib.HTTPMessage(self._file, seekable=0)
137
# Extract the range definition
138
content_range = self._headers.getheader('content-range', None)
139
if content_range is None:
140
raise errors.InvalidHttpResponse(
142
'Content-Range header missing in a multi-part response')
143
self.set_range_from_header(content_range)
145
def set_range_from_header(self, content_range):
146
"""Helper to set the new range from its description in the headers"""
148
rtype, values = content_range.split()
150
raise errors.InvalidHttpRange(self._path, content_range,
153
raise errors.InvalidHttpRange(self._path, content_range,
154
"Unsupported range type '%s'" % rtype)
156
# We don't need total, but note that it may be either the file size
157
# or '*' if the server can't or doesn't want to return the file
159
start_end, total = values.split('/')
160
start, end = start_end.split('-')
164
raise errors.InvalidHttpRange(self._path, content_range,
165
'Invalid range values')
166
size = end - start + 1
168
raise errors.InvalidHttpRange(self._path, content_range,
169
'Invalid range, size <= 0')
170
self.set_range(start, size)
172
def _checked_read(self, size):
173
"""Read the file checking for short reads.
175
The data read is discarded along the way.
180
data = self._file.read(min(remaining, self._discarded_buf_size))
181
remaining -= len(data)
183
raise errors.ShortReadvError(self._path, pos, size,
187
def _seek_to_next_range(self):
188
# We will cross range boundaries
189
if self._boundary is None:
190
# If we don't have a boundary, we can't find another range
191
raise errors.InvalidRange(self._path, self._pos,
192
"Range (%s, %s) exhausted"
193
% (self._start, self._size))
195
self.read_range_definition()
197
def read(self, size=-1):
198
"""Read size bytes from the current position in the file.
200
Reading across ranges is not supported. We rely on the underlying http
201
client to clean the socket if we leave bytes unread. This may occur for
202
the final boundary line of a multipart response or for any range
203
request not entirely consumed by the client (due to offset coalescing)
205
:param size: The number of bytes to read. Leave unspecified or pass
209
and self._pos == self._start + self._size):
213
self._seek_to_next_range()
214
elif self._pos < self._start:
215
raise errors.InvalidRange(
216
self._path, self._pos,
217
"Can't read %s bytes before range (%s, %s)"
218
% (size, self._start, self._size))
220
if size > 0 and self._pos + size > self._start + self._size:
221
raise errors.InvalidRange(
222
self._path, self._pos,
223
"Can't read %s bytes across range (%s, %s)"
224
% (size, self._start, self._size))
226
# read data from file
230
# Don't read past the range definition
231
limited = self._start + self._size - self._pos
233
limited = min(limited, size)
234
osutils.pumpfile(self._file, buffer, limited, self._max_read_size,
235
report_activity=self._report_activity, direction='read')
236
data = buffer.getvalue()
238
# Update _pos respecting the data effectively read
239
self._pos += len(data)
242
def seek(self, offset, whence=0):
243
start_pos = self._pos
247
final_pos = start_pos + offset
250
final_pos = self._start + self._size + offset # offset < 0
252
raise errors.InvalidRange(
253
self._path, self._pos,
254
"RangeFile: can't seek from end while size is unknown")
256
raise ValueError("Invalid value %s for whence." % whence)
258
if final_pos < self._pos:
259
# Can't seek backwards
260
raise errors.InvalidRange(
261
self._path, self._pos,
262
'RangeFile: trying to seek backwards to %s' % final_pos)
265
cur_limit = self._start + self._size
266
while final_pos > cur_limit:
267
# We will cross range boundaries
268
remain = cur_limit - self._pos
270
# Finish reading the current range
271
self._checked_read(remain)
272
self._seek_to_next_range()
273
cur_limit = self._start + self._size
275
size = final_pos - self._pos
276
if size > 0: # size can be < 0 if we crossed a range boundary
277
# We don't need the data, just read it and throw it away
278
self._checked_read(size)
284
def handle_response(url, code, msg, data, report_activity=None):
285
"""Interpret the code & headers and wrap the provided data in a RangeFile.
287
This is a factory method which returns an appropriate RangeFile based on
288
the code & headers it's given.
290
:param url: The url being processed. Mostly for error reporting
291
:param code: The integer HTTP response code
292
:param msg: An HTTPMessage containing the headers for the response
293
:param data: A file-like object that can be read() to get the
295
:return: A file-like object that can seek()+read() the
296
ranges indicated by the headers.
298
rfile = RangeFile(url, data, report_activity=report_activity)
301
size = msg.getheader('content-length', None)
306
rfile.set_range(0, size)
308
content_type = msg.getheader('content-type', None)
309
if content_type is None:
310
# When there is no content-type header we treat the response as
311
# being of type 'application/octet-stream' as per RFC2616 section
313
# Therefore it is obviously not multipart
314
content_type = 'application/octet-stream'
317
is_multipart = (msg.getmaintype() == 'multipart'
318
and msg.getsubtype() == 'byteranges')
321
# Full fledged multipart response
322
rfile.set_boundary(msg.getparam('boundary'))
324
# A response to a range request, but not multipart
325
content_range = msg.getheader('content-range', None)
326
if content_range is None:
327
raise errors.InvalidHttpResponse(url,
328
'Missing the Content-Range header in a 206 range response')
329
rfile.set_range_from_header(content_range)
331
raise errors.InvalidHttpResponse(url,
332
'Unknown response code %s' % code)
added
removed
bzrlib/transport/http/response.py
