To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
« back to all changes in this revision
Viewing changes to bzrlib/transport/http/__init__.py
- browse files at revision 2094.3.6
- compare with another revision
- compare with revision 5200.2.1
- download diff
- download tarball
- view history from revision 2094.3.6
- Committer: John Arbash Meinel
- Date: 2006-12-01 19:41:16 UTC
- mfrom: (2158 +trunk)
- mto: This revision was merged to the branch mainline in revision 2159.
- Revision ID: john@arbash-meinel.com-20061201194116-nvn5qhfxux5284jc
[merge] bzr.dev 2158
- files added:
- bzrlib/debug.py
- files modified:
- .bzrignore
added
removed
bzrlib/transport/http/__init__.py
20
20
"""
21
21
22
22
from cStringIO import StringIO
23
import errno
24
23
import mimetools
25
import os
26
import posixpath
27
24
import re
28
import sys
29
25
import urlparse
30
26
import urllib
31
from warnings import warn
32
33
# TODO: load these only when running http tests
34
import BaseHTTPServer, SimpleHTTPServer, socket, time
35
import threading
36
27
37
28
from bzrlib import errors, ui
38
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
39
TransportError, ConnectionError, InvalidURL)
40
from bzrlib.branch import Branch
41
29
from bzrlib.trace import mutter
42
30
from bzrlib.transport import (
43
get_transport,
44
register_transport,
45
Server,
46
31
smart,
47
32
Transport,
48
33
)
49
from bzrlib.transport.http.response import (HttpMultipartRangeResponse,
50
HttpRangeResponse)
51
52
34
35
36
# TODO: This is not used anymore by HttpTransport_urllib
37
# (extracting the auth info and prompting the user for a password
38
# have been split), only the tests still use it. It should be
39
# deleted and the tests rewritten ASAP to stay in sync.
53
40
def extract_auth(url, password_manager):
54
41
"""Extract auth parameters from am HTTP/HTTPS url and add them to the given
55
42
password manager. Return the url, minus those auth parameters (which
58
45
assert re.match(r'^(https?)(\+\w+)?://', url), \
59
46
'invalid absolute url %r' % url
60
47
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
61
48
62
49
if '@' in netloc:
63
50
auth, netloc = netloc.split('@', 1)
64
51
if ':' in auth:
73
60
if password is not None:
74
61
password = urllib.unquote(password)
75
62
else:
76
password = ui.ui_factory.get_password(prompt='HTTP %(user)@%(host) password',
77
user=username, host=host)
63
password = ui.ui_factory.get_password(
64
prompt='HTTP %(user)s@%(host)s password',
65
user=username, host=host)
78
66
password_manager.add_password(None, host, username, password)
79
67
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
80
68
return url
107
95
if not first_line.startswith('HTTP'):
108
96
if first_header: # The first header *must* start with HTTP
109
97
raise errors.InvalidHttpResponse(url,
110
'Opening header line did not start with HTTP: %s'
98
'Opening header line did not start with HTTP: %s'
111
99
% (first_line,))
112
100
assert False, 'Opening header line was not HTTP'
113
101
else:
136
124
# _proto: "http" or "https"
137
125
# _qualified_proto: may have "+pycurl", etc
138
126
139
def __init__(self, base):
127
def __init__(self, base, from_transport=None):
140
128
"""Set the base path where files will be stored."""
141
129
proto_match = re.match(r'^(https?)(\+\w+)?://', base)
142
130
if not proto_match:
149
137
if base[-1] != '/':
150
138
base = base + '/'
151
139
super(HttpTransportBase, self).__init__(base)
152
# In the future we might actually connect to the remote host
153
# rather than using get_url
154
# self._connection = None
155
140
(apparent_proto, self._host,
156
141
self._path, self._parameters,
157
142
self._query, self._fragment) = urlparse.urlparse(self.base)
158
143
self._qualified_proto = apparent_proto
144
# range hint is handled dynamically throughout the life
145
# of the object. We start by trying mulri-range requests
146
# and if the server returns bougs results, we retry with
147
# single range requests and, finally, we forget about
148
# range if the server really can't understand. Once
149
# aquired, this piece of info is propogated to clones.
150
if from_transport is not None:
151
self._range_hint = from_transport._range_hint
152
else:
153
self._range_hint = 'multi'
159
154
160
155
def abspath(self, relpath):
161
156
"""Return the full url to the given relative path.
168
163
"""
169
164
assert isinstance(relpath, basestring)
170
165
if isinstance(relpath, unicode):
171
raise InvalidURL(relpath, 'paths must not be unicode.')
166
raise errors.InvalidURL(relpath, 'paths must not be unicode.')
172
167
if isinstance(relpath, basestring):
173
168
relpath_parts = relpath.split('/')
174
169
else:
179
174
else:
180
175
# Except for the root, no trailing slashes are allowed
181
176
if len(relpath_parts) > 1 and relpath_parts[-1] == '':
182
raise ValueError("path %r within branch %r seems to be a directory"
183
% (relpath, self._path))
177
raise ValueError(
178
"path %r within branch %r seems to be a directory"
179
% (relpath, self._path))
184
180
basepath = self._path.split('/')
185
181
if len(basepath) > 0 and basepath[-1] == '':
186
182
basepath = basepath[:-1]
265
261
relpath, len(offsets), ranges)
266
262
code, f = self._get(relpath, ranges)
267
263
for start, size in offsets:
268
f.seek(start, (start < 0) and 2 or 0)
269
start = f.tell()
270
data = f.read(size)
271
if len(data) != size:
272
raise errors.ShortReadvError(relpath, start, size,
273
actual=len(data))
264
try_again = True
265
while try_again:
266
try_again = False
267
f.seek(start, (start < 0) and 2 or 0)
268
start = f.tell()
269
try:
270
data = f.read(size)
271
if len(data) != size:
272
raise errors.ShortReadvError(relpath, start, size,
273
actual=len(data))
274
except (errors.InvalidRange, errors.ShortReadvError):
275
# The server does not gives us enough data or
276
# bogus-looking result, let's try again with
277
# a simpler request if possible.
278
if self._range_hint == 'multi':
279
self._range_hint = 'single'
280
mutter('Retry %s with single range request' % relpath)
281
try_again = True
282
elif self._range_hint == 'single':
283
self._range_hint = None
284
mutter('Retry %s without ranges' % relpath)
285
try_again = True
286
if try_again:
287
# Note that since the offsets and the
288
# ranges may not be in the same order we
289
# dont't try to calculate a restricted
290
# single range encompassing unprocessed
291
# offsets. Note that we replace 'f' here
292
# and that it may need cleaning one day
293
# before being thrown that way.
294
code, f = self._get(relpath, ranges)
295
else:
296
# We tried all the tricks, nothing worked
297
raise
298
274
299
yield start, data
275
300
276
301
@staticmethod
317
342
:param relpath: Location to put the contents, relative to base.
318
343
:param f: File-like object.
319
344
"""
320
raise TransportNotPossible('http PUT not supported')
345
raise errors.TransportNotPossible('http PUT not supported')
321
346
322
347
def mkdir(self, relpath, mode=None):
323
348
"""Create a directory at the given path."""
324
raise TransportNotPossible('http does not support mkdir()')
349
raise errors.TransportNotPossible('http does not support mkdir()')
325
350
326
351
def rmdir(self, relpath):
327
352
"""See Transport.rmdir."""
328
raise TransportNotPossible('http does not support rmdir()')
353
raise errors.TransportNotPossible('http does not support rmdir()')
329
354
330
355
def append_file(self, relpath, f, mode=None):
331
356
"""Append the text in the file-like object into the final
332
357
location.
333
358
"""
334
raise TransportNotPossible('http does not support append()')
359
raise errors.TransportNotPossible('http does not support append()')
335
360
336
361
def copy(self, rel_from, rel_to):
337
362
"""Copy the item at rel_from to the location at rel_to"""
338
raise TransportNotPossible('http does not support copy()')
363
raise errors.TransportNotPossible('http does not support copy()')
339
364
340
365
def copy_to(self, relpaths, other, mode=None, pb=None):
341
366
"""Copy a set of entries from self into another Transport.
349
374
# the remote location is the same, and rather than download, and
350
375
# then upload, it could just issue a remote copy_this command.
351
376
if isinstance(other, HttpTransportBase):
352
raise TransportNotPossible('http cannot be the target of copy_to()')
377
raise errors.TransportNotPossible(
378
'http cannot be the target of copy_to()')
353
379
else:
354
380
return super(HttpTransportBase, self).\
355
381
copy_to(relpaths, other, mode=mode, pb=pb)
356
382
357
383
def move(self, rel_from, rel_to):
358
384
"""Move the item at rel_from to the location at rel_to"""
359
raise TransportNotPossible('http does not support move()')
385
raise errors.TransportNotPossible('http does not support move()')
360
386
361
387
def delete(self, relpath):
362
388
"""Delete the item at relpath"""
363
raise TransportNotPossible('http does not support delete()')
389
raise errors.TransportNotPossible('http does not support delete()')
364
390
365
391
def is_readonly(self):
366
392
"""See Transport.is_readonly."""
373
399
def stat(self, relpath):
374
400
"""Return the stat information for a file.
375
401
"""
376
raise TransportNotPossible('http does not support stat()')
402
raise errors.TransportNotPossible('http does not support stat()')
377
403
378
404
def lock_read(self, relpath):
379
405
"""Lock the given file for shared (read) access.
394
420
395
421
:return: A lock object, which should be passed to Transport.unlock()
396
422
"""
397
raise TransportNotPossible('http does not support lock_write()')
423
raise errors.TransportNotPossible('http does not support lock_write()')
398
424
399
425
def clone(self, offset=None):
400
426
"""Return a new HttpTransportBase with root at self.base + offset
407
433
else:
408
434
return self.__class__(self.abspath(offset), self)
409
435
436
def attempted_range_header(self, ranges, tail_amount):
437
"""Prepare a HTTP Range header at a level the server should accept"""
438
439
if self._range_hint == 'multi':
440
# Nothing to do here
441
return self.range_header(ranges, tail_amount)
442
elif self._range_hint == 'single':
443
# Combine all the requested ranges into a single
444
# encompassing one
445
if len(ranges) > 0:
446
start, ignored = ranges[0]
447
ignored, end = ranges[-1]
448
if tail_amount not in (0, None):
449
# Nothing we can do here to combine ranges
450
# with tail_amount, just returns None. The
451
# whole file should be downloaded.
452
return None
453
else:
454
return self.range_header([(start, end)], 0)
455
else:
456
# Only tail_amount, requested, leave range_header
457
# do its work
458
return self.range_header(ranges, tail_amount)
459
else:
460
return None
461
410
462
@staticmethod
411
463
def range_header(ranges, tail_amount):
412
464
"""Turn a list of bytes ranges into a HTTP Range header value.
413
465
414
:param offsets: A list of byte ranges, (start, end). An empty list
415
is not accepted.
466
:param ranges: A list of byte ranges, (start, end).
467
:param tail_amount: The amount to get from the end of the file.
416
468
417
469
:return: HTTP range header string.
470
471
At least a non-empty ranges *or* a tail_amount must be
472
provided.
418
473
"""
419
474
strings = []
420
475
for start, end in ranges:
447
502
448
503
def _read_bytes(self, count):
449
504
return self._response_body.read(count)
450
505
451
506
def _finished_reading(self):
452
507
"""See SmartClientMediumRequest._finished_reading."""
453
508
pass
454
455
456
#---------------- test server facilities ----------------
457
# TODO: load these only when running tests
458
459
460
class WebserverNotAvailable(Exception):
461
pass
462
463
464
class BadWebserverPath(ValueError):
465
def __str__(self):
466
return 'path %s is not in %s' % self.args
467
468
469
class TestingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
470
471
def log_message(self, format, *args):
472
self.server.test_case.log('webserver - %s - - [%s] %s "%s" "%s"',
473
self.address_string(),
474
self.log_date_time_string(),
475
format % args,
476
self.headers.get('referer', '-'),
477
self.headers.get('user-agent', '-'))
478
479
def handle_one_request(self):
480
"""Handle a single HTTP request.
481
482
You normally don't need to override this method; see the class
483
__doc__ string for information on how to handle specific HTTP
484
commands such as GET and POST.
485
486
"""
487
for i in xrange(1,11): # Don't try more than 10 times
488
try:
489
self.raw_requestline = self.rfile.readline()
490
except socket.error, e:
491
if e.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK):
492
# omitted for now because some tests look at the log of
493
# the server and expect to see no errors. see recent
494
# email thread. -- mbp 20051021.
495
## self.log_message('EAGAIN (%d) while reading from raw_requestline' % i)
496
time.sleep(0.01)
497
continue
498
raise
499
else:
500
break
501
if not self.raw_requestline:
502
self.close_connection = 1
503
return
504
if not self.parse_request(): # An error code has been sent, just exit
505
return
506
mname = 'do_' + self.command
507
if getattr(self, mname, None) is None:
508
self.send_error(501, "Unsupported method (%r)" % self.command)
509
return
510
method = getattr(self, mname)
511
method()
512
513
if sys.platform == 'win32':
514
# On win32 you cannot access non-ascii filenames without
515
# decoding them into unicode first.
516
# However, under Linux, you can access bytestream paths
517
# without any problems. If this function was always active
518
# it would probably break tests when LANG=C was set
519
def translate_path(self, path):
520
"""Translate a /-separated PATH to the local filename syntax.
521
522
For bzr, all url paths are considered to be utf8 paths.
523
On Linux, you can access these paths directly over the bytestream
524
request, but on win32, you must decode them, and access them
525
as Unicode files.
526
"""
527
# abandon query parameters
528
path = urlparse.urlparse(path)[2]
529
path = posixpath.normpath(urllib.unquote(path))
530
path = path.decode('utf-8')
531
words = path.split('/')
532
words = filter(None, words)
533
path = os.getcwdu()
534
for word in words:
535
drive, word = os.path.splitdrive(word)
536
head, word = os.path.split(word)
537
if word in (os.curdir, os.pardir): continue
538
path = os.path.join(path, word)
539
return path
540
541
542
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
543
def __init__(self, server_address, RequestHandlerClass, test_case):
544
BaseHTTPServer.HTTPServer.__init__(self, server_address,
545
RequestHandlerClass)
546
self.test_case = test_case
547
548
549
class HttpServer(Server):
550
"""A test server for http transports."""
551
552
# used to form the url that connects to this server
553
_url_protocol = 'http'
554
555
# Subclasses can provide a specific request handler
556
def __init__(self, request_handler=TestingHTTPRequestHandler):
557
Server.__init__(self)
558
self.request_handler = request_handler
559
560
def _get_httpd(self):
561
return TestingHTTPServer(('localhost', 0),
562
self.request_handler,
563
self)
564
565
def _http_start(self):
566
httpd = self._get_httpd()
567
host, port = httpd.socket.getsockname()
568
self._http_base_url = '%s://localhost:%s/' % (self._url_protocol, port)
569
self._http_starting.release()
570
httpd.socket.settimeout(0.1)
571
572
while self._http_running:
573
try:
574
httpd.handle_request()
575
except socket.timeout:
576
pass
577
578
def _get_remote_url(self, path):
579
path_parts = path.split(os.path.sep)
580
if os.path.isabs(path):
581
if path_parts[:len(self._local_path_parts)] != \
582
self._local_path_parts:
583
raise BadWebserverPath(path, self.test_dir)
584
remote_path = '/'.join(path_parts[len(self._local_path_parts):])
585
else:
586
remote_path = '/'.join(path_parts)
587
588
self._http_starting.acquire()
589
self._http_starting.release()
590
return self._http_base_url + remote_path
591
592
def log(self, format, *args):
593
"""Capture Server log output."""
594
self.logs.append(format % args)
595
596
def setUp(self):
597
"""See bzrlib.transport.Server.setUp."""
598
self._home_dir = os.getcwdu()
599
self._local_path_parts = self._home_dir.split(os.path.sep)
600
self._http_starting = threading.Lock()
601
self._http_starting.acquire()
602
self._http_running = True
603
self._http_base_url = None
604
self._http_thread = threading.Thread(target=self._http_start)
605
self._http_thread.setDaemon(True)
606
self._http_thread.start()
607
self._http_proxy = os.environ.get("http_proxy")
608
if self._http_proxy is not None:
609
del os.environ["http_proxy"]
610
self.logs = []
611
612
def tearDown(self):
613
"""See bzrlib.transport.Server.tearDown."""
614
self._http_running = False
615
self._http_thread.join()
616
if self._http_proxy is not None:
617
import os
618
os.environ["http_proxy"] = self._http_proxy
619
620
def get_url(self):
621
"""See bzrlib.transport.Server.get_url."""
622
return self._get_remote_url(self._home_dir)
623
624
def get_bogus_url(self):
625
"""See bzrlib.transport.Server.get_bogus_url."""
626
# this is chosen to try to prevent trouble with proxies, weird dns,
627
# etc
628
return 'http://127.0.0.1:1/'
629
630
631
class HTTPServerWithSmarts(HttpServer):
632
"""HTTPServerWithSmarts extends the HttpServer with POST methods that will
633
trigger a smart server to execute with a transport rooted at the rootdir of
634
the HTTP server.
635
"""
636
637
def __init__(self):
638
HttpServer.__init__(self, SmartRequestHandler)
639
640
641
class SmartRequestHandler(TestingHTTPRequestHandler):
642
"""Extend TestingHTTPRequestHandler to support smart client POSTs."""
643
644
def do_POST(self):
645
"""Hand the request off to a smart server instance."""
646
self.send_response(200)
647
self.send_header("Content-type", "application/octet-stream")
648
transport = get_transport(self.server.test_case._home_dir)
649
# TODO: We might like to support streaming responses. 1.0 allows no
650
# Content-length in this case, so for integrity we should perform our
651
# own chunking within the stream.
652
# 1.1 allows chunked responses, and in this case we could chunk using
653
# the HTTP chunking as this will allow HTTP persistence safely, even if
654
# we have to stop early due to error, but we would also have to use the
655
# HTTP trailer facility which may not be widely available.
656
out_buffer = StringIO()
657
smart_protocol_request = smart.SmartServerRequestProtocolOne(
658
transport, out_buffer.write)
659
# if this fails, we should return 400 bad request, but failure is
660
# failure for now - RBC 20060919
661
data_length = int(self.headers['Content-Length'])
662
# Perhaps there should be a SmartServerHTTPMedium that takes care of
663
# feeding the bytes in the http request to the smart_protocol_request,
664
# but for now it's simpler to just feed the bytes directly.
665
smart_protocol_request.accept_bytes(self.rfile.read(data_length))
666
assert smart_protocol_request.next_read_size() == 0, (
667
"not finished reading, but all data sent to protocol.")
668
self.send_header("Content-Length", str(len(out_buffer.getvalue())))
669
self.end_headers()
670
self.wfile.write(out_buffer.getvalue())
671
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0