1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'SmartTCPTransport', 'SmartSSHTransport']
25
from cStringIO import StringIO
33
from bzrlib.smart import client, medium, protocol
35
# must do this otherwise urllib can't parse the urls properly :(
36
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
37
transport.register_urlparse_netloc_protocol(scheme)
41
class _SmartStat(object):
43
def __init__(self, size, mode):
48
class RemoteTransport(transport.Transport):
49
"""Connection to a smart server.
51
The connection holds references to the medium that can be used to send
52
requests to the server.
54
The connection has a notion of the current directory to which it's
55
connected; this is incorporated in filenames passed to the server.
57
This supports some higher-level RPC operations and can also be treated
58
like a Transport to do file-like operations.
60
The connection can be made over a tcp socket, an ssh pipe or a series of
61
http requests. There are concrete subclasses for each type:
62
SmartTCPTransport, etc.
65
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
66
# responsibilities: Put those on SmartClient or similar. This is vital for
67
# the ability to support multiple versions of the smart protocol over time:
68
# RemoteTransport is an adapter from the Transport object model to the
69
# SmartClient model, not an encoder.
71
def __init__(self, url, clone_from=None, medium=None):
74
:param medium: The medium to use for this RemoteTransport. This must be
75
supplied if clone_from is None.
77
### Technically super() here is faulty because Transport's __init__
78
### fails to take 2 parameters, and if super were to choose a silly
79
### initialisation order things would blow up.
80
if not url.endswith('/'):
82
super(RemoteTransport, self).__init__(url)
83
self._scheme, self._username, self._password, self._host, self._port, self._path = \
84
transport.split_url(url)
85
if clone_from is None:
88
# credentials may be stripped from the base in some circumstances
89
# as yet to be clearly defined or documented, so copy them.
90
self._username = clone_from._username
91
# reuse same connection
92
self._medium = clone_from._medium
93
assert self._medium is not None
95
def abspath(self, relpath):
96
"""Return the full url to the given relative path.
98
@param relpath: the relative path or path components
99
@type relpath: str or list
101
return self._unparse_url(self._remote_path(relpath))
103
def clone(self, relative_url):
104
"""Make a new RemoteTransport related to me, sharing the same connection.
106
This essentially opens a handle on a different remote directory.
108
if relative_url is None:
109
return RemoteTransport(self.base, self)
111
return RemoteTransport(self.abspath(relative_url), self)
113
def is_readonly(self):
114
"""Smart server transport can do read/write file operations."""
117
def get_smart_client(self):
120
def get_smart_medium(self):
123
def _unparse_url(self, path):
124
"""Return URL for a path.
126
:see: SFTPUrlHandling._unparse_url
128
# TODO: Eventually it should be possible to unify this with
129
# SFTPUrlHandling._unparse_url?
132
path = urllib.quote(path)
133
netloc = urllib.quote(self._host)
134
if self._username is not None:
135
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
136
if self._port is not None:
137
netloc = '%s:%d' % (netloc, self._port)
138
return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))
140
def _remote_path(self, relpath):
141
"""Returns the Unicode version of the absolute path for relpath."""
142
return self._combine_paths(self._path, relpath)
144
def _call(self, method, *args):
145
resp = self._call2(method, *args)
146
self._translate_error(resp)
148
def _call2(self, method, *args):
149
"""Call a method on the remote server."""
150
return client.SmartClient(self._medium).call(method, *args)
152
def _call_with_body_bytes(self, method, args, body):
153
"""Call a method on the remote server with body bytes."""
154
smart_client = client.SmartClient(self._medium)
155
return smart_client.call_with_body_bytes(method, args, body)
157
def has(self, relpath):
158
"""Indicate whether a remote file of the given name exists or not.
160
:see: Transport.has()
162
resp = self._call2('has', self._remote_path(relpath))
163
if resp == ('yes', ):
165
elif resp == ('no', ):
168
self._translate_error(resp)
170
def get(self, relpath):
171
"""Return file-like object reading the contents of a remote file.
173
:see: Transport.get_bytes()/get_file()
175
return StringIO(self.get_bytes(relpath))
177
def get_bytes(self, relpath):
178
remote = self._remote_path(relpath)
179
request = self._medium.get_request()
180
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
181
smart_protocol.call('get', remote)
182
resp = smart_protocol.read_response_tuple(True)
184
smart_protocol.cancel_read_body()
185
self._translate_error(resp, relpath)
186
return smart_protocol.read_body_bytes()
188
def _serialise_optional_mode(self, mode):
194
def mkdir(self, relpath, mode=None):
195
resp = self._call2('mkdir', self._remote_path(relpath),
196
self._serialise_optional_mode(mode))
197
self._translate_error(resp)
199
def put_bytes(self, relpath, upload_contents, mode=None):
200
# FIXME: upload_file is probably not safe for non-ascii characters -
201
# should probably just pass all parameters as length-delimited
203
resp = self._call_with_body_bytes('put',
204
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
206
self._translate_error(resp)
208
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
209
create_parent_dir=False,
211
"""See Transport.put_bytes_non_atomic."""
212
# FIXME: no encoding in the transport!
213
create_parent_str = 'F'
214
if create_parent_dir:
215
create_parent_str = 'T'
217
resp = self._call_with_body_bytes(
219
(self._remote_path(relpath), self._serialise_optional_mode(mode),
220
create_parent_str, self._serialise_optional_mode(dir_mode)),
222
self._translate_error(resp)
224
def put_file(self, relpath, upload_file, mode=None):
225
# its not ideal to seek back, but currently put_non_atomic_file depends
226
# on transports not reading before failing - which is a faulty
227
# assumption I think - RBC 20060915
228
pos = upload_file.tell()
230
return self.put_bytes(relpath, upload_file.read(), mode)
232
upload_file.seek(pos)
235
def put_file_non_atomic(self, relpath, f, mode=None,
236
create_parent_dir=False,
238
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
239
create_parent_dir=create_parent_dir,
242
def append_file(self, relpath, from_file, mode=None):
243
return self.append_bytes(relpath, from_file.read(), mode)
245
def append_bytes(self, relpath, bytes, mode=None):
246
resp = self._call_with_body_bytes(
248
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
250
if resp[0] == 'appended':
252
self._translate_error(resp)
254
def delete(self, relpath):
255
resp = self._call2('delete', self._remote_path(relpath))
256
self._translate_error(resp)
258
def readv(self, relpath, offsets):
262
offsets = list(offsets)
264
sorted_offsets = sorted(offsets)
265
# turn the list of offsets into a stack
266
offset_stack = iter(offsets)
267
cur_offset_and_size = offset_stack.next()
268
coalesced = list(self._coalesce_offsets(sorted_offsets,
269
limit=self._max_readv_combine,
270
fudge_factor=self._bytes_to_read_before_seek))
272
request = self._medium.get_request()
273
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
274
smart_protocol.call_with_body_readv_array(
275
('readv', self._remote_path(relpath)),
276
[(c.start, c.length) for c in coalesced])
277
resp = smart_protocol.read_response_tuple(True)
279
if resp[0] != 'readv':
280
# This should raise an exception
281
smart_protocol.cancel_read_body()
282
self._translate_error(resp)
285
# FIXME: this should know how many bytes are needed, for clarity.
286
data = smart_protocol.read_body_bytes()
287
# Cache the results, but only until they have been fulfilled
289
for c_offset in coalesced:
290
if len(data) < c_offset.length:
291
raise errors.ShortReadvError(relpath, c_offset.start,
292
c_offset.length, actual=len(data))
293
for suboffset, subsize in c_offset.ranges:
294
key = (c_offset.start+suboffset, subsize)
295
data_map[key] = data[suboffset:suboffset+subsize]
296
data = data[c_offset.length:]
298
# Now that we've read some data, see if we can yield anything back
299
while cur_offset_and_size in data_map:
300
this_data = data_map.pop(cur_offset_and_size)
301
yield cur_offset_and_size[0], this_data
302
cur_offset_and_size = offset_stack.next()
304
def rename(self, rel_from, rel_to):
306
self._remote_path(rel_from),
307
self._remote_path(rel_to))
309
def move(self, rel_from, rel_to):
311
self._remote_path(rel_from),
312
self._remote_path(rel_to))
314
def rmdir(self, relpath):
315
resp = self._call('rmdir', self._remote_path(relpath))
317
def _translate_error(self, resp, orig_path=None):
318
"""Raise an exception from a response"""
325
elif what == 'NoSuchFile':
326
if orig_path is not None:
327
error_path = orig_path
330
raise errors.NoSuchFile(error_path)
331
elif what == 'error':
332
raise errors.SmartProtocolError(unicode(resp[1]))
333
elif what == 'FileExists':
334
raise errors.FileExists(resp[1])
335
elif what == 'DirectoryNotEmpty':
336
raise errors.DirectoryNotEmpty(resp[1])
337
elif what == 'ShortReadvError':
338
raise errors.ShortReadvError(resp[1], int(resp[2]),
339
int(resp[3]), int(resp[4]))
340
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
341
encoding = str(resp[1]) # encoding must always be a string
345
reason = str(resp[5]) # reason must always be a string
346
if val.startswith('u:'):
347
val = val[2:].decode('utf-8')
348
elif val.startswith('s:'):
349
val = val[2:].decode('base64')
350
if what == 'UnicodeDecodeError':
351
raise UnicodeDecodeError(encoding, val, start, end, reason)
352
elif what == 'UnicodeEncodeError':
353
raise UnicodeEncodeError(encoding, val, start, end, reason)
354
elif what == "ReadOnlyError":
355
raise errors.TransportNotPossible('readonly transport')
357
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
359
def disconnect(self):
360
self._medium.disconnect()
362
def delete_tree(self, relpath):
363
raise errors.TransportNotPossible('readonly transport')
365
def stat(self, relpath):
366
resp = self._call2('stat', self._remote_path(relpath))
367
if resp[0] == 'stat':
368
return _SmartStat(int(resp[1]), int(resp[2], 8))
370
self._translate_error(resp)
372
## def lock_read(self, relpath):
373
## """Lock the given file for shared (read) access.
374
## :return: A lock object, which should be passed to Transport.unlock()
376
## # The old RemoteBranch ignore lock for reading, so we will
377
## # continue that tradition and return a bogus lock object.
378
## class BogusLock(object):
379
## def __init__(self, path):
383
## return BogusLock(relpath)
388
def list_dir(self, relpath):
389
resp = self._call2('list_dir', self._remote_path(relpath))
390
if resp[0] == 'names':
391
return [name.encode('ascii') for name in resp[1:]]
393
self._translate_error(resp)
395
def iter_files_recursive(self):
396
resp = self._call2('iter_files_recursive', self._remote_path(''))
397
if resp[0] == 'names':
400
self._translate_error(resp)
403
class SmartTCPTransport(RemoteTransport):
404
"""Connection to smart server over plain tcp.
406
This is essentially just a factory to get 'RemoteTransport(url,
407
SmartTCPClientMedium).
410
def __init__(self, url):
411
_scheme, _username, _password, _host, _port, _path = \
412
transport.split_url(url)
415
except (ValueError, TypeError), e:
416
raise errors.InvalidURL(path=url, extra="invalid port %s" % _port)
417
client_medium = medium.SmartTCPClientMedium(_host, _port)
418
super(SmartTCPTransport, self).__init__(url, medium=client_medium)
421
class SmartSSHTransport(RemoteTransport):
422
"""Connection to smart server over SSH.
424
This is essentially just a factory to get 'RemoteTransport(url,
425
SmartSSHClientMedium).
428
def __init__(self, url):
429
_scheme, _username, _password, _host, _port, _path = \
430
transport.split_url(url)
432
if _port is not None:
434
except (ValueError, TypeError), e:
435
raise errors.InvalidURL(path=url, extra="invalid port %s" %
437
client_medium = medium.SmartSSHClientMedium(_host, _port,
438
_username, _password)
439
super(SmartSSHTransport, self).__init__(url, medium=client_medium)
442
class SmartHTTPTransport(RemoteTransport):
443
"""Just a way to connect between a bzr+http:// url and http://.
445
This connection operates slightly differently than the SmartSSHTransport.
446
It uses a plain http:// transport underneath, which defines what remote
447
.bzr/smart URL we are connected to. From there, all paths that are sent are
448
sent as relative paths, this way, the remote side can properly
449
de-reference them, since it is likely doing rewrite rules to translate an
450
HTTP path into a local path.
453
def __init__(self, url, http_transport=None):
454
assert url.startswith('bzr+http://')
456
if http_transport is None:
457
http_url = url[len('bzr+'):]
458
self._http_transport = transport.get_transport(http_url)
460
self._http_transport = http_transport
461
http_medium = self._http_transport.get_smart_medium()
462
super(SmartHTTPTransport, self).__init__(url, medium=http_medium)
464
def _remote_path(self, relpath):
465
"""After connecting HTTP Transport only deals in relative URLs."""
471
def abspath(self, relpath):
472
"""Return the full url to the given relative path.
474
:param relpath: the relative path or path components
475
:type relpath: str or list
477
return self._unparse_url(self._combine_paths(self._path, relpath))
479
def clone(self, relative_url):
480
"""Make a new SmartHTTPTransport related to me.
482
This is re-implemented rather than using the default
483
SmartTransport.clone() because we must be careful about the underlying
487
abs_url = self.abspath(relative_url)
490
# By cloning the underlying http_transport, we are able to share the
492
new_transport = self._http_transport.clone(relative_url)
493
return SmartHTTPTransport(abs_url, http_transport=new_transport)
496
def get_test_permutations():
497
"""Return (transport, server) permutations for testing."""
498
### We may need a little more test framework support to construct an
499
### appropriate RemoteTransport in the future.
500
from bzrlib.smart import server
501
return [(SmartTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
