19
19
There are separate implementation modules for each http client implementation.
22
from cStringIO import StringIO
24
from collections import deque
25
from cStringIO import StringIO
29
31
from warnings import warn
31
from bzrlib.transport import Transport, register_transport, Server
33
# TODO: load these only when running http tests
34
import BaseHTTPServer, SimpleHTTPServer, socket, time
37
from bzrlib import errors
32
38
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
33
39
TransportError, ConnectionError, InvalidURL)
34
40
from bzrlib.branch import Branch
35
41
from bzrlib.trace import mutter
36
# TODO: load these only when running http tests
37
import BaseHTTPServer, SimpleHTTPServer, socket, time
42
from bzrlib.transport import Transport, register_transport, Server
43
from bzrlib.transport.http.response import (HttpMultipartRangeResponse,
39
45
from bzrlib.ui import ui_factory
78
def _extract_headers(header_text, url):
79
"""Extract the mapping for an rfc2822 header
81
This is a helper function for the test suite and for _pycurl.
82
(urllib already parses the headers for us)
84
In the case that there are multiple headers inside the file,
85
the last one is returned.
87
:param header_text: A string of header information.
88
This expects that the first line of a header will always be HTTP ...
89
:param url: The url we are parsing, so we can raise nice errors
90
:return: mimetools.Message object, which basically acts like a case
91
insensitive dictionary.
94
remaining = header_text
97
raise errors.InvalidHttpResponse(url, 'Empty headers')
100
header_file = StringIO(remaining)
101
first_line = header_file.readline()
102
if not first_line.startswith('HTTP'):
103
if first_header: # The first header *must* start with HTTP
104
raise errors.InvalidHttpResponse(url,
105
'Opening header line did not start with HTTP: %s'
107
assert False, 'Opening header line was not HTTP'
109
break # We are done parsing
111
m = mimetools.Message(header_file)
113
# mimetools.Message parses the first header up to a blank line
114
# So while there is remaining data, it probably means there is
115
# another header to be parsed.
116
# Get rid of any preceeding whitespace, which if it is all whitespace
117
# will get rid of everything.
118
remaining = header_file.read().lstrip()
72
122
class HttpTransportBase(Transport):
73
123
"""Base class for http implementations.
193
243
:param offsets: A list of (offset, size) tuples.
194
244
:param return: A list or generator of (offset, data) tuples
196
# Ideally we would pass one big request asking for all the ranges in
197
# one go; however then the server will give a multipart mime response
198
# back, and we can't parse them yet. So instead we just get one range
199
# per region, and try to coallesce the regions as much as possible.
201
# The read-coallescing code is not quite regular enough to have a
202
# single driver routine and
203
# helper method in Transport.
204
def do_combined_read(combined_offsets):
205
# read one coalesced block
207
for offset, size in combined_offsets:
209
mutter('readv coalesced %d reads.', len(combined_offsets))
210
offset = combined_offsets[0][0]
211
byte_range = (offset, offset + total_size - 1)
212
code, result_file = self._get(relpath, [byte_range])
214
for off, size in combined_offsets:
215
result_bytes = result_file.read(size)
216
assert len(result_bytes) == size
217
yield off, result_bytes
219
data = result_file.read(offset + total_size)[offset:offset + total_size]
221
for offset, size in combined_offsets:
222
yield offset, data[pos:pos + size]
227
pending_offsets = deque(offsets)
228
combined_offsets = []
229
while len(pending_offsets):
230
offset, size = pending_offsets.popleft()
231
if not combined_offsets:
232
combined_offsets = [[offset, size]]
246
ranges = self.offsets_to_ranges(offsets)
247
mutter('http readv of %s collapsed %s offsets => %s',
248
relpath, len(offsets), ranges)
249
code, f = self._get(relpath, ranges)
250
for start, size in offsets:
251
f.seek(start, (start < 0) and 2 or 0)
254
assert len(data) == size
258
def offsets_to_ranges(offsets):
259
"""Turn a list of offsets and sizes into a list of byte ranges.
261
:param offsets: A list of tuples of (start, size). An empty list
263
:return: a list of inclusive byte ranges (start, end)
264
Adjacent ranges will be combined.
266
# Make sure we process sorted offsets
267
offsets = sorted(offsets)
272
for start, size in offsets:
273
end = start + size - 1
275
combined.append([start, end])
276
elif start <= prev_end + 1:
277
combined[-1][1] = end
234
if (len (combined_offsets) < 500 and
235
combined_offsets[-1][0] + combined_offsets[-1][1] == offset):
237
combined_offsets.append([offset, size])
239
# incompatible, or over the threshold issue a read and yield
240
pending_offsets.appendleft((offset, size))
241
for result in do_combined_read(combined_offsets):
243
combined_offsets = []
244
# whatever is left is a single coalesced request
245
if len(combined_offsets):
246
for result in do_combined_read(combined_offsets):
279
combined.append([start, end])
249
284
def put(self, relpath, f, mode=None):
250
285
"""Copy the file-like or string object into the location.
342
377
return self.__class__(self.abspath(offset))
380
def range_header(ranges, tail_amount):
381
"""Turn a list of bytes ranges into a HTTP Range header value.
383
:param offsets: A list of byte ranges, (start, end). An empty list
386
:return: HTTP range header string.
389
for start, end in ranges:
390
strings.append('%d-%d' % (start, end))
393
strings.append('-%d' % tail_amount)
395
return ','.join(strings)
344
398
#---------------- test server facilities ----------------
345
399
# TODO: load these only when running tests
398
452
method = getattr(self, mname)
455
if sys.platform == 'win32':
456
# On win32 you cannot access non-ascii filenames without
457
# decoding them into unicode first.
458
# However, under Linux, you can access bytestream paths
459
# without any problems. If this function was always active
460
# it would probably break tests when LANG=C was set
461
def translate_path(self, path):
462
"""Translate a /-separated PATH to the local filename syntax.
464
For bzr, all url paths are considered to be utf8 paths.
465
On Linux, you can access these paths directly over the bytestream
466
request, but on win32, you must decode them, and access them
469
# abandon query parameters
470
path = urlparse.urlparse(path)[2]
471
path = posixpath.normpath(urllib.unquote(path))
472
path = path.decode('utf-8')
473
words = path.split('/')
474
words = filter(None, words)
477
drive, word = os.path.splitdrive(word)
478
head, word = os.path.split(word)
479
if word in (os.curdir, os.pardir): continue
480
path = os.path.join(path, word)
402
484
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
403
485
def __init__(self, server_address, RequestHandlerClass, test_case):
added
removed
bzrlib/transport/http/__init__.py
