1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
34
from bzrlib.smart import client, medium, protocol
36
# must do this otherwise urllib can't parse the urls properly :(
37
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
38
transport.register_urlparse_netloc_protocol(scheme)
42
# Port 4155 is the default port for bzr://, registered with IANA.
43
BZR_DEFAULT_INTERFACE = '0.0.0.0'
44
BZR_DEFAULT_PORT = 4155
47
class _SmartStat(object):
49
def __init__(self, size, mode):
54
class RemoteTransport(transport.Transport):
55
"""Connection to a smart server.
57
The connection holds references to the medium that can be used to send
58
requests to the server.
60
The connection has a notion of the current directory to which it's
61
connected; this is incorporated in filenames passed to the server.
63
This supports some higher-level RPC operations and can also be treated
64
like a Transport to do file-like operations.
66
The connection can be made over a tcp socket, an ssh pipe or a series of
67
http requests. There are concrete subclasses for each type:
68
RemoteTCPTransport, etc.
71
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
72
# responsibilities: Put those on SmartClient or similar. This is vital for
73
# the ability to support multiple versions of the smart protocol over time:
74
# RemoteTransport is an adapter from the Transport object model to the
75
# SmartClient model, not an encoder.
77
def __init__(self, url, clone_from=None, medium=None, _client=None):
80
:param clone_from: Another RemoteTransport instance that this one is
81
being cloned from. Attributes such as credentials and the medium
83
:param medium: The medium to use for this RemoteTransport. This must be
84
supplied if clone_from is None.
85
:param _client: Override the _SmartClient used by this transport. This
86
should only be used for testing purposes; normally this is
87
determined from the medium.
89
### Technically super() here is faulty because Transport's __init__
90
### fails to take 2 parameters, and if super were to choose a silly
91
### initialisation order things would blow up.
92
if not url.endswith('/'):
94
super(RemoteTransport, self).__init__(url)
95
self._scheme, self._username, self._password, self._host, self._port, self._path = \
96
transport.split_url(url)
97
if clone_from is None:
100
# credentials may be stripped from the base in some circumstances
101
# as yet to be clearly defined or documented, so copy them.
102
self._username = clone_from._username
103
# reuse same connection
104
self._medium = clone_from._medium
105
assert self._medium is not None
107
self._client = client._SmartClient(self._medium)
109
self._client = _client
111
def abspath(self, relpath):
112
"""Return the full url to the given relative path.
114
@param relpath: the relative path or path components
115
@type relpath: str or list
117
return self._unparse_url(self._remote_path(relpath))
119
def clone(self, relative_url):
120
"""Make a new RemoteTransport related to me, sharing the same connection.
122
This essentially opens a handle on a different remote directory.
124
if relative_url is None:
125
return RemoteTransport(self.base, self)
127
return RemoteTransport(self.abspath(relative_url), self)
129
def is_readonly(self):
130
"""Smart server transport can do read/write file operations."""
131
resp = self._call2('Transport.is_readonly')
132
if resp == ('yes', ):
134
elif resp == ('no', ):
136
elif resp == ('error', "Generic bzr smart protocol error: "
137
"bad request 'Transport.is_readonly'"):
138
# XXX: nasty hack: servers before 0.16 don't have a
139
# 'Transport.is_readonly' verb, so we do what clients before 0.16
143
self._translate_error(resp)
144
assert False, 'weird response %r' % (resp,)
146
def get_smart_client(self):
149
def get_smart_medium(self):
152
def _unparse_url(self, path):
153
"""Return URL for a path.
155
:see: SFTPUrlHandling._unparse_url
157
# TODO: Eventually it should be possible to unify this with
158
# SFTPUrlHandling._unparse_url?
161
path = urllib.quote(path)
162
netloc = urllib.quote(self._host)
163
if self._username is not None:
164
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
165
if self._port is not None:
166
netloc = '%s:%d' % (netloc, self._port)
167
return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))
169
def _remote_path(self, relpath):
170
"""Returns the Unicode version of the absolute path for relpath."""
171
return self._combine_paths(self._path, relpath)
173
def _call(self, method, *args):
174
resp = self._call2(method, *args)
175
self._translate_error(resp)
177
def _call2(self, method, *args):
178
"""Call a method on the remote server."""
179
return self._client.call(method, *args)
181
def _call_with_body_bytes(self, method, args, body):
182
"""Call a method on the remote server with body bytes."""
183
return self._client.call_with_body_bytes(method, args, body)
185
def has(self, relpath):
186
"""Indicate whether a remote file of the given name exists or not.
188
:see: Transport.has()
190
resp = self._call2('has', self._remote_path(relpath))
191
if resp == ('yes', ):
193
elif resp == ('no', ):
196
self._translate_error(resp)
198
def get(self, relpath):
199
"""Return file-like object reading the contents of a remote file.
201
:see: Transport.get_bytes()/get_file()
203
return StringIO(self.get_bytes(relpath))
205
def get_bytes(self, relpath):
206
remote = self._remote_path(relpath)
207
request = self._medium.get_request()
208
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
209
smart_protocol.call('get', remote)
210
resp = smart_protocol.read_response_tuple(True)
212
smart_protocol.cancel_read_body()
213
self._translate_error(resp, relpath)
214
return smart_protocol.read_body_bytes()
216
def _serialise_optional_mode(self, mode):
222
def mkdir(self, relpath, mode=None):
223
resp = self._call2('mkdir', self._remote_path(relpath),
224
self._serialise_optional_mode(mode))
225
self._translate_error(resp)
227
def put_bytes(self, relpath, upload_contents, mode=None):
228
# FIXME: upload_file is probably not safe for non-ascii characters -
229
# should probably just pass all parameters as length-delimited
231
if type(upload_contents) is unicode:
232
# Although not strictly correct, we raise UnicodeEncodeError to be
233
# compatible with other transports.
234
raise UnicodeEncodeError(
235
'undefined', upload_contents, 0, 1,
236
'put_bytes must be given bytes, not unicode.')
237
resp = self._call_with_body_bytes('put',
238
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
240
self._translate_error(resp)
242
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
243
create_parent_dir=False,
245
"""See Transport.put_bytes_non_atomic."""
246
# FIXME: no encoding in the transport!
247
create_parent_str = 'F'
248
if create_parent_dir:
249
create_parent_str = 'T'
251
resp = self._call_with_body_bytes(
253
(self._remote_path(relpath), self._serialise_optional_mode(mode),
254
create_parent_str, self._serialise_optional_mode(dir_mode)),
256
self._translate_error(resp)
258
def put_file(self, relpath, upload_file, mode=None):
259
# its not ideal to seek back, but currently put_non_atomic_file depends
260
# on transports not reading before failing - which is a faulty
261
# assumption I think - RBC 20060915
262
pos = upload_file.tell()
264
return self.put_bytes(relpath, upload_file.read(), mode)
266
upload_file.seek(pos)
269
def put_file_non_atomic(self, relpath, f, mode=None,
270
create_parent_dir=False,
272
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
273
create_parent_dir=create_parent_dir,
276
def append_file(self, relpath, from_file, mode=None):
277
return self.append_bytes(relpath, from_file.read(), mode)
279
def append_bytes(self, relpath, bytes, mode=None):
280
resp = self._call_with_body_bytes(
282
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
284
if resp[0] == 'appended':
286
self._translate_error(resp)
288
def delete(self, relpath):
289
resp = self._call2('delete', self._remote_path(relpath))
290
self._translate_error(resp)
292
def readv(self, relpath, offsets):
296
offsets = list(offsets)
298
sorted_offsets = sorted(offsets)
299
# turn the list of offsets into a stack
300
offset_stack = iter(offsets)
301
cur_offset_and_size = offset_stack.next()
302
coalesced = list(self._coalesce_offsets(sorted_offsets,
303
limit=self._max_readv_combine,
304
fudge_factor=self._bytes_to_read_before_seek))
306
request = self._medium.get_request()
307
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
308
smart_protocol.call_with_body_readv_array(
309
('readv', self._remote_path(relpath)),
310
[(c.start, c.length) for c in coalesced])
311
resp = smart_protocol.read_response_tuple(True)
313
if resp[0] != 'readv':
314
# This should raise an exception
315
smart_protocol.cancel_read_body()
316
self._translate_error(resp)
319
# FIXME: this should know how many bytes are needed, for clarity.
320
data = smart_protocol.read_body_bytes()
321
# Cache the results, but only until they have been fulfilled
323
for c_offset in coalesced:
324
if len(data) < c_offset.length:
325
raise errors.ShortReadvError(relpath, c_offset.start,
326
c_offset.length, actual=len(data))
327
for suboffset, subsize in c_offset.ranges:
328
key = (c_offset.start+suboffset, subsize)
329
data_map[key] = data[suboffset:suboffset+subsize]
330
data = data[c_offset.length:]
332
# Now that we've read some data, see if we can yield anything back
333
while cur_offset_and_size in data_map:
334
this_data = data_map.pop(cur_offset_and_size)
335
yield cur_offset_and_size[0], this_data
336
cur_offset_and_size = offset_stack.next()
338
def rename(self, rel_from, rel_to):
340
self._remote_path(rel_from),
341
self._remote_path(rel_to))
343
def move(self, rel_from, rel_to):
345
self._remote_path(rel_from),
346
self._remote_path(rel_to))
348
def rmdir(self, relpath):
349
resp = self._call('rmdir', self._remote_path(relpath))
351
def _translate_error(self, resp, orig_path=None):
352
"""Raise an exception from a response"""
359
elif what == 'NoSuchFile':
360
if orig_path is not None:
361
error_path = orig_path
364
raise errors.NoSuchFile(error_path)
365
elif what == 'error':
366
raise errors.SmartProtocolError(unicode(resp[1]))
367
elif what == 'FileExists':
368
raise errors.FileExists(resp[1])
369
elif what == 'DirectoryNotEmpty':
370
raise errors.DirectoryNotEmpty(resp[1])
371
elif what == 'ShortReadvError':
372
raise errors.ShortReadvError(resp[1], int(resp[2]),
373
int(resp[3]), int(resp[4]))
374
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
375
encoding = str(resp[1]) # encoding must always be a string
379
reason = str(resp[5]) # reason must always be a string
380
if val.startswith('u:'):
381
val = val[2:].decode('utf-8')
382
elif val.startswith('s:'):
383
val = val[2:].decode('base64')
384
if what == 'UnicodeDecodeError':
385
raise UnicodeDecodeError(encoding, val, start, end, reason)
386
elif what == 'UnicodeEncodeError':
387
raise UnicodeEncodeError(encoding, val, start, end, reason)
388
elif what == "ReadOnlyError":
389
raise errors.TransportNotPossible('readonly transport')
391
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
393
def disconnect(self):
394
self._medium.disconnect()
396
def delete_tree(self, relpath):
397
raise errors.TransportNotPossible('readonly transport')
399
def stat(self, relpath):
400
resp = self._call2('stat', self._remote_path(relpath))
401
if resp[0] == 'stat':
402
return _SmartStat(int(resp[1]), int(resp[2], 8))
404
self._translate_error(resp)
406
## def lock_read(self, relpath):
407
## """Lock the given file for shared (read) access.
408
## :return: A lock object, which should be passed to Transport.unlock()
410
## # The old RemoteBranch ignore lock for reading, so we will
411
## # continue that tradition and return a bogus lock object.
412
## class BogusLock(object):
413
## def __init__(self, path):
417
## return BogusLock(relpath)
422
def list_dir(self, relpath):
423
resp = self._call2('list_dir', self._remote_path(relpath))
424
if resp[0] == 'names':
425
return [name.encode('ascii') for name in resp[1:]]
427
self._translate_error(resp)
429
def iter_files_recursive(self):
430
resp = self._call2('iter_files_recursive', self._remote_path(''))
431
if resp[0] == 'names':
434
self._translate_error(resp)
437
class RemoteTCPTransport(RemoteTransport):
438
"""Connection to smart server over plain tcp.
440
This is essentially just a factory to get 'RemoteTransport(url,
441
SmartTCPClientMedium).
444
def __init__(self, url):
445
_scheme, _username, _password, _host, _port, _path = \
446
transport.split_url(url)
448
_port = BZR_DEFAULT_PORT
452
except (ValueError, TypeError), e:
453
raise errors.InvalidURL(
454
path=url, extra="invalid port %s" % _port)
455
client_medium = medium.SmartTCPClientMedium(_host, _port)
456
super(RemoteTCPTransport, self).__init__(url, medium=client_medium)
459
class RemoteSSHTransport(RemoteTransport):
460
"""Connection to smart server over SSH.
462
This is essentially just a factory to get 'RemoteTransport(url,
463
SmartSSHClientMedium).
466
def __init__(self, url):
467
_scheme, _username, _password, _host, _port, _path = \
468
transport.split_url(url)
470
if _port is not None:
472
except (ValueError, TypeError), e:
473
raise errors.InvalidURL(path=url, extra="invalid port %s" %
475
client_medium = medium.SmartSSHClientMedium(_host, _port,
476
_username, _password)
477
super(RemoteSSHTransport, self).__init__(url, medium=client_medium)
480
class RemoteHTTPTransport(RemoteTransport):
481
"""Just a way to connect between a bzr+http:// url and http://.
483
This connection operates slightly differently than the RemoteSSHTransport.
484
It uses a plain http:// transport underneath, which defines what remote
485
.bzr/smart URL we are connected to. From there, all paths that are sent are
486
sent as relative paths, this way, the remote side can properly
487
de-reference them, since it is likely doing rewrite rules to translate an
488
HTTP path into a local path.
491
def __init__(self, url, http_transport=None):
492
assert url.startswith('bzr+http://')
494
if http_transport is None:
495
http_url = url[len('bzr+'):]
496
self._http_transport = transport.get_transport(http_url)
498
self._http_transport = http_transport
499
http_medium = self._http_transport.get_smart_medium()
500
super(RemoteHTTPTransport, self).__init__(url, medium=http_medium)
502
def _remote_path(self, relpath):
503
"""After connecting HTTP Transport only deals in relative URLs."""
504
# Adjust the relpath based on which URL this smart transport is
506
base = urlutils.normalize_url(self._http_transport.base)
507
url = urlutils.join(self.base[len('bzr+'):], relpath)
508
url = urlutils.normalize_url(url)
509
return urlutils.relative_url(base, url)
511
def abspath(self, relpath):
512
"""Return the full url to the given relative path.
514
:param relpath: the relative path or path components
515
:type relpath: str or list
517
return self._unparse_url(self._combine_paths(self._path, relpath))
519
def clone(self, relative_url):
520
"""Make a new RemoteHTTPTransport related to me.
522
This is re-implemented rather than using the default
523
RemoteTransport.clone() because we must be careful about the underlying
526
Also, the cloned smart transport will POST to the same .bzr/smart
527
location as this transport (although obviously the relative paths in the
528
smart requests may be different). This is so that the server doesn't
529
have to handle .bzr/smart requests at arbitrary places inside .bzr
530
directories, just at the initial URL the user uses.
532
The exception is parent paths (i.e. relative_url of "..").
535
abs_url = self.abspath(relative_url)
538
# We either use the exact same http_transport (for child locations), or
539
# a clone of the underlying http_transport (for parent locations). This
540
# means we share the connection.
541
norm_base = urlutils.normalize_url(self.base)
542
norm_abs_url = urlutils.normalize_url(abs_url)
543
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
544
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
545
http_transport = self._http_transport.clone(normalized_rel_url)
547
http_transport = self._http_transport
548
return RemoteHTTPTransport(abs_url, http_transport=http_transport)
551
def get_test_permutations():
552
"""Return (transport, server) permutations for testing."""
553
### We may need a little more test framework support to construct an
554
### appropriate RemoteTransport in the future.
555
from bzrlib.smart import server
556
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
