13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25
17
from cStringIO import StringIO
27
21
from bzrlib import (
36
from bzrlib.smart import client, medium
37
from bzrlib.symbol_versioning import (
42
class _SmartStat(object):
25
from bzrlib.smart.protocol import SmartClientRequestProtocolOne
26
from bzrlib.smart.medium import SmartTCPClientMedium, SmartSSHClientMedium
28
# must do this otherwise urllib can't parse the urls properly :(
29
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
30
transport.register_urlparse_netloc_protocol(scheme)
34
# Port 4155 is the default port for bzr://, registered with IANA.
35
BZR_DEFAULT_PORT = 4155
38
class SmartStat(object):
44
40
def __init__(self, size, mode):
45
41
self.st_size = size
46
42
self.st_mode = mode
49
class RemoteTransport(transport.ConnectedTransport):
45
class RemoteTransport(transport.Transport):
50
46
"""Connection to a smart server.
52
The connection holds references to the medium that can be used to send
53
requests to the server.
48
The connection holds references to pipes that can be used to send requests
55
51
The connection has a notion of the current directory to which it's
56
52
connected; this is incorporated in filenames passed to the server.
58
This supports some higher-level RPC operations and can also be treated
54
This supports some higher-level RPC operations and can also be treated
59
55
like a Transport to do file-like operations.
61
The connection can be made over a tcp socket, an ssh pipe or a series of
62
http requests. There are concrete subclasses for each type:
63
RemoteTCPTransport, etc.
57
The connection can be made over a tcp socket, or (in future) an ssh pipe
58
or a series of http requests. There are concrete subclasses for each
59
type: RemoteTCPTransport, etc.
66
# When making a readv request, cap it at requesting 5MB of data
67
_max_readv_bytes = 5*1024*1024
69
62
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
70
63
# responsibilities: Put those on SmartClient or similar. This is vital for
71
64
# the ability to support multiple versions of the smart protocol over time:
72
# RemoteTransport is an adapter from the Transport object model to the
65
# RemoteTransport is an adapter from the Transport object model to the
73
66
# SmartClient model, not an encoder.
75
# FIXME: the medium parameter should be private, only the tests requires
76
# it. It may be even clearer to define a TestRemoteTransport that handles
77
# the specific cases of providing a _client and/or a _medium, and leave
78
# RemoteTransport as an abstract class.
79
def __init__(self, url, _from_transport=None, medium=None, _client=None):
68
def __init__(self, url, clone_from=None, medium=None):
82
:param _from_transport: Another RemoteTransport instance that this
83
one is being cloned from. Attributes such as the medium will
86
:param medium: The medium to use for this RemoteTransport. If None,
87
the medium from the _from_transport is shared. If both this
88
and _from_transport are None, a new medium will be built.
89
_from_transport and medium cannot both be specified.
91
:param _client: Override the _SmartClient used by this transport. This
92
should only be used for testing purposes; normally this is
93
determined from the medium.
95
super(RemoteTransport, self).__init__(
96
url, _from_transport=_from_transport)
98
# The medium is the connection, except when we need to share it with
99
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
100
# what we want to share is really the shared connection.
102
if (_from_transport is not None
103
and isinstance(_from_transport, RemoteTransport)):
104
_client = _from_transport._client
105
elif _from_transport is None:
106
# If no _from_transport is specified, we need to intialize the
110
medium, credentials = self._build_medium()
111
if 'hpss' in debug.debug_flags:
112
trace.mutter('hpss: Built a new medium: %s',
113
medium.__class__.__name__)
114
self._shared_connection = transport._SharedConnection(medium,
118
# No medium was specified, so share the medium from the
120
medium = self._shared_connection.connection
122
raise AssertionError(
123
"Both _from_transport (%r) and medium (%r) passed to "
124
"RemoteTransport.__init__, but these parameters are mutally "
125
"exclusive." % (_from_transport, medium))
128
self._client = client._SmartClient(medium)
130
self._client = _client
132
def _build_medium(self):
133
"""Create the medium if _from_transport does not provide one.
135
The medium is analogous to the connection for ConnectedTransport: it
136
allows connection sharing.
141
def _report_activity(self, bytes, direction):
142
"""See Transport._report_activity.
144
Does nothing; the smart medium will report activity triggered by a
71
:param medium: The medium to use for this RemoteTransport. This must be
72
supplied if clone_from is None.
74
### Technically super() here is faulty because Transport's __init__
75
### fails to take 2 parameters, and if super were to choose a silly
76
### initialisation order things would blow up.
77
if not url.endswith('/'):
79
super(RemoteTransport, self).__init__(url)
80
self._scheme, self._username, self._password, self._host, self._port, self._path = \
81
transport.split_url(url)
82
if clone_from is None:
85
# credentials may be stripped from the base in some circumstances
86
# as yet to be clearly defined or documented, so copy them.
87
self._username = clone_from._username
88
# reuse same connection
89
self._medium = clone_from._medium
90
assert self._medium is not None
92
def abspath(self, relpath):
93
"""Return the full url to the given relative path.
95
@param relpath: the relative path or path components
96
@type relpath: str or list
98
return self._unparse_url(self._remote_path(relpath))
100
def clone(self, relative_url):
101
"""Make a new RemoteTransport related to me, sharing the same connection.
103
This essentially opens a handle on a different remote directory.
105
if relative_url is None:
106
return RemoteTransport(self.base, self)
108
return RemoteTransport(self.abspath(relative_url), self)
149
110
def is_readonly(self):
150
111
"""Smart server transport can do read/write file operations."""
152
resp = self._call2('Transport.is_readonly')
153
except errors.UnknownSmartMethod:
154
# XXX: nasty hack: servers before 0.16 don't have a
155
# 'Transport.is_readonly' verb, so we do what clients before 0.16
158
if resp == ('yes', ):
160
elif resp == ('no', ):
163
raise errors.UnexpectedSmartServerResponse(resp)
165
114
def get_smart_client(self):
166
return self._get_connection()
168
117
def get_smart_medium(self):
169
return self._get_connection()
120
def _unparse_url(self, path):
121
"""Return URL for a path.
123
:see: SFTPUrlHandling._unparse_url
125
# TODO: Eventually it should be possible to unify this with
126
# SFTPUrlHandling._unparse_url?
129
path = urllib.quote(path)
130
netloc = urllib.quote(self._host)
131
if self._username is not None:
132
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
133
if self._port is not None:
134
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))
171
137
def _remote_path(self, relpath):
172
138
"""Returns the Unicode version of the absolute path for relpath."""
175
141
def _call(self, method, *args):
176
142
resp = self._call2(method, *args)
177
self._ensure_ok(resp)
143
self._translate_error(resp)
179
145
def _call2(self, method, *args):
180
146
"""Call a method on the remote server."""
182
return self._client.call(method, *args)
183
except errors.ErrorFromSmartServer, err:
184
# The first argument, if present, is always a path.
186
context = {'relpath': args[0]}
189
self._translate_error(err, **context)
147
protocol = SmartClientRequestProtocolOne(self._medium.get_request())
148
protocol.call(method, *args)
149
return protocol.read_response_tuple()
191
151
def _call_with_body_bytes(self, method, args, body):
192
152
"""Call a method on the remote server with body bytes."""
194
return self._client.call_with_body_bytes(method, args, body)
195
except errors.ErrorFromSmartServer, err:
196
# The first argument, if present, is always a path.
198
context = {'relpath': args[0]}
201
self._translate_error(err, **context)
153
protocol = SmartClientRequestProtocolOne(self._medium.get_request())
154
protocol.call_with_body_bytes((method, ) + args, body)
155
return protocol.read_response_tuple()
203
157
def has(self, relpath):
204
158
"""Indicate whether a remote file of the given name exists or not.
240
193
def mkdir(self, relpath, mode=None):
241
194
resp = self._call2('mkdir', self._remote_path(relpath),
242
195
self._serialise_optional_mode(mode))
244
def open_write_stream(self, relpath, mode=None):
245
"""See Transport.open_write_stream."""
246
self.put_bytes(relpath, "", mode)
247
result = transport.AppendBasedFileStream(self, relpath)
248
transport._file_streams[self.abspath(relpath)] = result
196
self._translate_error(resp)
251
198
def put_bytes(self, relpath, upload_contents, mode=None):
252
199
# FIXME: upload_file is probably not safe for non-ascii characters -
253
200
# should probably just pass all parameters as length-delimited
255
if type(upload_contents) is unicode:
256
# Although not strictly correct, we raise UnicodeEncodeError to be
257
# compatible with other transports.
258
raise UnicodeEncodeError(
259
'undefined', upload_contents, 0, 1,
260
'put_bytes must be given bytes, not unicode.')
261
202
resp = self._call_with_body_bytes('put',
262
203
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
264
self._ensure_ok(resp)
265
return len(upload_contents)
205
self._translate_error(resp)
267
207
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
268
208
create_parent_dir=False,
309
249
if resp[0] == 'appended':
310
250
return int(resp[1])
311
raise errors.UnexpectedSmartServerResponse(resp)
251
self._translate_error(resp)
313
253
def delete(self, relpath):
314
254
resp = self._call2('delete', self._remote_path(relpath))
315
self._ensure_ok(resp)
317
def external_url(self):
318
"""See bzrlib.transport.Transport.external_url."""
319
# the external path for RemoteTransports is the base
322
def recommended_page_size(self):
323
"""Return the recommended page size for this transport."""
326
def _readv(self, relpath, offsets):
255
self._translate_error(resp)
257
def readv(self, relpath, offsets):
330
261
offsets = list(offsets)
332
263
sorted_offsets = sorted(offsets)
264
# turn the list of offsets into a stack
265
offset_stack = iter(offsets)
266
cur_offset_and_size = offset_stack.next()
333
267
coalesced = list(self._coalesce_offsets(sorted_offsets,
334
268
limit=self._max_readv_combine,
335
fudge_factor=self._bytes_to_read_before_seek,
336
max_size=self._max_readv_bytes))
338
# now that we've coallesced things, avoid making enormous requests
343
if c.length + cur_len > self._max_readv_bytes:
344
requests.append(cur_request)
348
cur_request.append(c)
351
requests.append(cur_request)
352
if 'hpss' in debug.debug_flags:
353
trace.mutter('%s.readv %s offsets => %s coalesced'
354
' => %s requests (%s)',
355
self.__class__.__name__, len(offsets), len(coalesced),
356
len(requests), sum(map(len, requests)))
269
fudge_factor=self._bytes_to_read_before_seek))
271
protocol = SmartClientRequestProtocolOne(self._medium.get_request())
272
protocol.call_with_body_readv_array(
273
('readv', self._remote_path(relpath)),
274
[(c.start, c.length) for c in coalesced])
275
resp = protocol.read_response_tuple(True)
277
if resp[0] != 'readv':
278
# This should raise an exception
279
protocol.cancel_read_body()
280
self._translate_error(resp)
283
# FIXME: this should know how many bytes are needed, for clarity.
284
data = protocol.read_body_bytes()
357
285
# Cache the results, but only until they have been fulfilled
359
# turn the list of offsets into a single stack to iterate
360
offset_stack = iter(offsets)
361
# using a list so it can be modified when passing down and coming back
362
next_offset = [offset_stack.next()]
363
for cur_request in requests:
365
result = self._client.call_with_body_readv_array(
366
('readv', self._remote_path(relpath),),
367
[(c.start, c.length) for c in cur_request])
368
resp, response_handler = result
369
except errors.ErrorFromSmartServer, err:
370
self._translate_error(err, relpath)
372
if resp[0] != 'readv':
373
# This should raise an exception
374
response_handler.cancel_read_body()
375
raise errors.UnexpectedSmartServerResponse(resp)
377
for res in self._handle_response(offset_stack, cur_request,
383
def _handle_response(self, offset_stack, coalesced, response_handler,
384
data_map, next_offset):
385
cur_offset_and_size = next_offset[0]
386
# FIXME: this should know how many bytes are needed, for clarity.
387
data = response_handler.read_body_bytes()
389
287
for c_offset in coalesced:
390
288
if len(data) < c_offset.length:
391
289
raise errors.ShortReadvError(relpath, c_offset.start,
392
290
c_offset.length, actual=len(data))
393
291
for suboffset, subsize in c_offset.ranges:
394
292
key = (c_offset.start+suboffset, subsize)
395
this_data = data[data_offset+suboffset:
396
data_offset+suboffset+subsize]
397
# Special case when the data is in-order, rather than packing
398
# into a map and then back out again. Benchmarking shows that
399
# this has 100% hit rate, but leave in the data_map work just
401
# TODO: Could we get away with using buffer() to avoid the
402
# memory copy? Callers would need to realize they may
403
# not have a real string.
404
if key == cur_offset_and_size:
405
yield cur_offset_and_size[0], this_data
406
cur_offset_and_size = next_offset[0] = offset_stack.next()
408
data_map[key] = this_data
409
data_offset += c_offset.length
293
data_map[key] = data[suboffset:suboffset+subsize]
294
data = data[c_offset.length:]
411
296
# Now that we've read some data, see if we can yield anything back
412
297
while cur_offset_and_size in data_map:
413
298
this_data = data_map.pop(cur_offset_and_size)
414
299
yield cur_offset_and_size[0], this_data
415
cur_offset_and_size = next_offset[0] = offset_stack.next()
300
cur_offset_and_size = offset_stack.next()
417
302
def rename(self, rel_from, rel_to):
418
303
self._call('rename',
427
312
def rmdir(self, relpath):
428
313
resp = self._call('rmdir', self._remote_path(relpath))
430
def _ensure_ok(self, resp):
432
raise errors.UnexpectedSmartServerResponse(resp)
434
def _translate_error(self, err, relpath=None):
435
remote._translate_error(err, path=relpath)
315
def _translate_error(self, resp, orig_path=None):
316
"""Raise an exception from a response"""
323
elif what == 'NoSuchFile':
324
if orig_path is not None:
325
error_path = orig_path
328
raise errors.NoSuchFile(error_path)
329
elif what == 'error':
330
raise errors.SmartProtocolError(unicode(resp[1]))
331
elif what == 'FileExists':
332
raise errors.FileExists(resp[1])
333
elif what == 'DirectoryNotEmpty':
334
raise errors.DirectoryNotEmpty(resp[1])
335
elif what == 'ShortReadvError':
336
raise errors.ShortReadvError(resp[1], int(resp[2]),
337
int(resp[3]), int(resp[4]))
338
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
339
encoding = str(resp[1]) # encoding must always be a string
343
reason = str(resp[5]) # reason must always be a string
344
if val.startswith('u:'):
345
val = val[2:].decode('utf-8')
346
elif val.startswith('s:'):
347
val = val[2:].decode('base64')
348
if what == 'UnicodeDecodeError':
349
raise UnicodeDecodeError(encoding, val, start, end, reason)
350
elif what == 'UnicodeEncodeError':
351
raise UnicodeEncodeError(encoding, val, start, end, reason)
352
elif what == "ReadOnlyError":
353
raise errors.TransportNotPossible('readonly transport')
355
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
437
357
def disconnect(self):
438
m = self.get_smart_medium()
358
self._medium.disconnect()
360
def delete_tree(self, relpath):
361
raise errors.TransportNotPossible('readonly transport')
442
363
def stat(self, relpath):
443
364
resp = self._call2('stat', self._remote_path(relpath))
444
365
if resp[0] == 'stat':
445
return _SmartStat(int(resp[1]), int(resp[2], 8))
446
raise errors.UnexpectedSmartServerResponse(resp)
366
return SmartStat(int(resp[1]), int(resp[2], 8))
368
self._translate_error(resp)
448
370
## def lock_read(self, relpath):
449
371
## """Lock the given file for shared (read) access.
465
387
resp = self._call2('list_dir', self._remote_path(relpath))
466
388
if resp[0] == 'names':
467
389
return [name.encode('ascii') for name in resp[1:]]
468
raise errors.UnexpectedSmartServerResponse(resp)
391
self._translate_error(resp)
470
393
def iter_files_recursive(self):
471
394
resp = self._call2('iter_files_recursive', self._remote_path(''))
472
395
if resp[0] == 'names':
474
raise errors.UnexpectedSmartServerResponse(resp)
398
self._translate_error(resp)
477
402
class RemoteTCPTransport(RemoteTransport):
478
403
"""Connection to smart server over plain tcp.
480
405
This is essentially just a factory to get 'RemoteTransport(url,
481
406
SmartTCPClientMedium).
484
def _build_medium(self):
485
client_medium = medium.SmartTCPClientMedium(
486
self._host, self._port, self.base)
487
return client_medium, None
490
class RemoteTCPTransportV2Only(RemoteTransport):
491
"""Connection to smart server over plain tcp with the client hard-coded to
492
assume protocol v2 and remote server version <= 1.6.
494
This should only be used for testing.
497
def _build_medium(self):
498
client_medium = medium.SmartTCPClientMedium(
499
self._host, self._port, self.base)
500
client_medium._protocol_version = 2
501
client_medium._remember_remote_is_before((1, 6))
502
return client_medium, None
409
def __init__(self, url):
410
_scheme, _username, _password, _host, _port, _path = \
411
transport.split_url(url)
413
_port = BZR_DEFAULT_PORT
417
except (ValueError, TypeError), e:
418
raise errors.InvalidURL(
419
path=url, extra="invalid port %s" % _port)
420
medium = SmartTCPClientMedium(_host, _port)
421
super(RemoteTCPTransport, self).__init__(url, medium=medium)
505
424
class RemoteSSHTransport(RemoteTransport):
533
452
HTTP path into a local path.
536
def __init__(self, base, _from_transport=None, http_transport=None):
455
def __init__(self, url, http_transport=None):
456
assert url.startswith('bzr+http://')
537
458
if http_transport is None:
538
# FIXME: the password may be lost here because it appears in the
539
# url only for an intial construction (when the url came from the
541
http_url = base[len('bzr+'):]
459
http_url = url[len('bzr+'):]
542
460
self._http_transport = transport.get_transport(http_url)
544
462
self._http_transport = http_transport
545
super(RemoteHTTPTransport, self).__init__(
546
base, _from_transport=_from_transport)
548
def _build_medium(self):
549
# We let http_transport take care of the credentials
550
return self._http_transport.get_smart_medium(), None
463
http_medium = self._http_transport.get_smart_medium()
464
super(RemoteHTTPTransport, self).__init__(url, medium=http_medium)
552
466
def _remote_path(self, relpath):
553
"""After connecting, HTTP Transport only deals in relative URLs."""
554
# Adjust the relpath based on which URL this smart transport is
556
http_base = urlutils.normalize_url(self.get_smart_medium().base)
557
url = urlutils.join(self.base[len('bzr+'):], relpath)
558
url = urlutils.normalize_url(url)
559
return urlutils.relative_url(http_base, url)
467
"""After connecting HTTP Transport only deals in relative URLs."""
473
def abspath(self, relpath):
474
"""Return the full url to the given relative path.
476
:param relpath: the relative path or path components
477
:type relpath: str or list
479
return self._unparse_url(self._combine_paths(self._path, relpath))
561
481
def clone(self, relative_url):
562
482
"""Make a new RemoteHTTPTransport related to me.
564
484
This is re-implemented rather than using the default
565
485
RemoteTransport.clone() because we must be careful about the underlying
568
Also, the cloned smart transport will POST to the same .bzr/smart
569
location as this transport (although obviously the relative paths in the
570
smart requests may be different). This is so that the server doesn't
571
have to handle .bzr/smart requests at arbitrary places inside .bzr
572
directories, just at the initial URL the user uses.
575
489
abs_url = self.abspath(relative_url)
577
491
abs_url = self.base
578
return RemoteHTTPTransport(abs_url,
579
_from_transport=self,
580
http_transport=self._http_transport)
582
def _redirected_to(self, source, target):
583
"""See transport._redirected_to"""
584
redirected = self._http_transport._redirected_to(source, target)
585
if (redirected is not None
586
and isinstance(redirected, type(self._http_transport))):
587
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
588
http_transport=redirected)
590
# Either None or a transport for a different protocol
594
class HintingSSHTransport(transport.Transport):
595
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
597
def __init__(self, url):
598
raise errors.UnsupportedProtocol(url,
599
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
492
# By cloning the underlying http_transport, we are able to share the
494
new_transport = self._http_transport.clone(relative_url)
495
return RemoteHTTPTransport(abs_url, http_transport=new_transport)
602
498
def get_test_permutations():
603
499
"""Return (transport, server) permutations for testing."""
500
from bzrlib.smart import server
604
501
### We may need a little more test framework support to construct an
605
502
### appropriate RemoteTransport in the future.
606
from bzrlib.tests import test_server
607
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
503
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
