14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
import urllib, urllib2
20
from StringIO import StringIO
17
from cStringIO import StringIO
22
import bzrlib # for the version
23
from bzrlib.errors import (TransportNotPossible, NoSuchFile, BzrError,
24
TransportError, ConnectionError)
21
from bzrlib import errors
25
22
from bzrlib.trace import mutter
26
23
from bzrlib.transport import register_urlparse_netloc_protocol
27
from bzrlib.transport.http import (HttpTransportBase, HttpServer,
28
extract_auth, response)
24
from bzrlib.transport.http import HttpTransportBase
25
# TODO: handle_response should be integrated into the _urllib2_wrappers
26
from bzrlib.transport.http.response import handle_response
27
from bzrlib.transport.http._urllib2_wrappers import (
30
extract_authentication_uri,
30
35
register_urlparse_netloc_protocol('http+urllib')
33
class Request(urllib2.Request):
34
"""Request object for urllib2 that allows the method to be overridden."""
39
if self.method is not None:
42
return urllib2.Request.get_method(self)
45
38
class HttpTransport_urllib(HttpTransportBase):
46
39
"""Python urllib transport for http and https."""
48
# TODO: Implement pipelined versions of all of the *_multi() functions.
41
# In order to debug we have to issue our traces in sync with
42
# httplib, which use print :(
45
_opener_class = Opener
50
47
def __init__(self, base, from_transport=None):
51
48
"""Set the base path where files will be stored."""
52
super(HttpTransport_urllib, self).__init__(base)
53
# HttpTransport_urllib doesn't maintain any per-transport state yet
54
# so nothing to do with from_transport
49
if from_transport is not None:
50
super(HttpTransport_urllib, self).__init__(base, from_transport)
51
self._connection = from_transport._connection
52
self._auth = from_transport._auth
53
self._proxy_auth = from_transport._proxy_auth
55
self._opener = from_transport._opener
57
# urllib2 will be confused if it find authentication
58
# info in the urls. So we handle them separatly.
59
# Note: we don't need to when cloning because it was
61
clean_base, user, password = extract_credentials(base)
62
super(HttpTransport_urllib, self).__init__(clean_base,
64
self._connection = None
65
self._opener = self._opener_class()
67
authuri = extract_authentication_uri(self._real_abspath(self._path))
68
self._auth = {'user': user, 'password': password,
70
if user and password is not None: # '' is a valid password
71
# Make the (user, password) available to urllib2
72
# We default to a realm of None to catch them all.
73
self._opener.password_manager.add_password(None, authuri,
77
def _perform(self, request):
78
"""Send the request to the server and handles common errors.
80
:returns: urllib2 Response object
82
if self._connection is not None:
83
# Give back shared info
84
request.connection = self._connection
85
# Ensure authentication info is provided
86
request.auth = self._auth
87
request.proxy_auth = self._proxy_auth
89
mutter('%s: [%s]' % (request.method, request.get_full_url()))
90
if self._debuglevel > 0:
91
print 'perform: %s base: %s, url: %s' % (request.method, self.base,
92
request.get_full_url())
93
response = self._opener.open(request)
94
if self._connection is None:
95
# Acquire connection when the first request is able
96
# to connect to the server
97
self._connection = request.connection
98
# Always get auth parameters, they may change
99
self._auth = request.auth
100
self._proxy_auth = request.proxy_auth
103
if request.follow_redirections is False \
104
and code in (301, 302, 303, 307):
105
raise errors.RedirectRequested(request.get_full_url(),
106
request.redirected_to,
107
is_permament=(code == 301),
108
qual_proto=self._qualified_proto)
110
if request.redirected_to is not None:
111
mutter('redirected from: %s to: %s' % (request.get_full_url(),
112
request.redirected_to))
56
116
def _get(self, relpath, ranges, tail_amount=0):
59
path = self._real_abspath(relpath)
60
resp = self._get_url_impl(path, method='GET', ranges=ranges,
61
tail_amount=tail_amount)
62
return resp.code, response.handle_response(path,
63
resp.code, resp.headers, resp)
64
except urllib2.HTTPError, e:
65
mutter('url error code: %s for has url: %r', e.code, path)
67
raise NoSuchFile(path, extra=e)
69
except (BzrError, IOError), e:
70
if getattr(e, 'errno', None) is not None:
71
mutter('io error: %s %s for has url: %r',
72
e.errno, errno.errorcode.get(e.errno), path)
73
if e.errno == errno.ENOENT:
74
raise NoSuchFile(path, extra=e)
75
raise ConnectionError(msg = "Error retrieving %s: %s"
76
% (self.abspath(relpath), str(e)),
79
def _get_url_impl(self, url, method, ranges, tail_amount=0):
80
"""Actually pass get request into urllib
82
:returns: urllib Response object
84
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
85
url = extract_auth(url, manager)
86
auth_handler = urllib2.HTTPBasicAuthHandler(manager)
87
opener = urllib2.build_opener(auth_handler)
88
request = Request(url)
89
request.method = method
90
request.add_header('Pragma', 'no-cache')
91
request.add_header('Cache-control', 'max-age=0')
92
request.add_header('User-Agent',
93
'bzr/%s (urllib)' % (bzrlib.__version__,))
117
"""See HttpTransport._get"""
119
abspath = self._real_abspath(relpath)
94
121
if ranges or tail_amount:
95
bytes = 'bytes=' + self.range_header(ranges, tail_amount)
96
request.add_header('Range', bytes)
97
response = opener.open(request)
122
range_header = self.attempted_range_header(ranges, tail_amount)
123
if range_header is not None:
124
bytes = 'bytes=' + range_header
125
headers = {'Range': bytes}
127
request = Request('GET', abspath, None, headers)
128
response = self._perform(request)
131
if code == 404: # not found
132
self._connection.fake_close()
133
raise errors.NoSuchFile(abspath)
135
data = handle_response(abspath, code, response.headers, response)
136
# Close response to free the httplib.HTTPConnection pipeline
137
self._connection.fake_close()
140
def _post(self, body_bytes):
141
abspath = self._real_abspath('.bzr/smart')
142
response = self._perform(Request('POST', abspath, body_bytes))
144
data = handle_response(abspath, code, response.headers, response)
145
# Close response to free the httplib.HTTPConnection pipeline
146
self._connection.fake_close()
100
149
def should_cache(self):
101
150
"""Return True if the data pulled across should be cached locally.
154
def _head(self, relpath):
155
"""Request the HEAD of a file.
157
Performs the request and leaves callers handle the results.
159
abspath = self._real_abspath(relpath)
160
request = Request('HEAD', abspath)
161
response = self._perform(request)
163
self._connection.fake_close()
105
166
def has(self, relpath):
106
167
"""Does the target location exist?
108
abspath = self._real_abspath(relpath)
110
f = self._get_url_impl(abspath, 'HEAD', [])
111
# Without the read and then close()
112
# we tend to have busy sockets.
169
response = self._head(relpath)
172
if code == 200: # "ok",
116
except urllib2.HTTPError, e:
117
mutter('url error code: %s, for has url: %r', e.code, abspath)
121
except urllib2.URLError, e:
122
mutter('url error: %s, for has url: %r', e.reason, abspath)
125
mutter('io error: %s %s for has url: %r',
126
e.errno, errno.errorcode.get(e.errno), abspath)
127
if e.errno == errno.ENOENT:
129
raise TransportError(orig_error=e)
131
def copy_to(self, relpaths, other, mode=None, pb=None):
132
"""Copy a set of entries from self into another Transport.
134
:param relpaths: A list/generator of entries to be copied.
136
TODO: if other is LocalTransport, is it possible to
137
do better than put(get())?
139
# At this point HttpTransport_urllib might be able to check and see if
140
# the remote location is the same, and rather than download, and
141
# then upload, it could just issue a remote copy_this command.
142
if isinstance(other, HttpTransport_urllib):
143
raise TransportNotPossible('http cannot be the target of copy_to()')
145
return super(HttpTransport_urllib, self).copy_to(relpaths, other, mode=mode, pb=pb)
147
def move(self, rel_from, rel_to):
148
"""Move the item at rel_from to the location at rel_to"""
149
raise TransportNotPossible('http does not support move()')
151
def delete(self, relpath):
152
"""Delete the item at relpath"""
153
raise TransportNotPossible('http does not support delete()')
156
class HttpServer_urllib(HttpServer):
157
"""Subclass of HttpServer that gives http+urllib urls.
159
This is for use in testing: connections to this server will always go
160
through urllib where possible.
163
# urls returned by this server should require the urllib client impl
164
_url_protocol = 'http+urllib'
175
assert(code == 404, 'Only 200 or 404 are correct')
167
179
def get_test_permutations():
168
180
"""Return the permutations to be used in testing."""
181
from bzrlib.tests.HttpServer import HttpServer_urllib
169
182
return [(HttpTransport_urllib, HttpServer_urllib),
added
removed
bzrlib/transport/http/_urllib.py
