23
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
25
from cStringIO import StringIO
27
29
from bzrlib import (
36
from bzrlib.smart import client, medium
37
from bzrlib.symbol_versioning import (
37
from bzrlib.smart import client, medium, protocol
39
# must do this otherwise urllib can't parse the urls properly :(
40
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http', 'bzr+https']:
41
transport.register_urlparse_netloc_protocol(scheme)
45
# Port 4155 is the default port for bzr://, registered with IANA.
46
BZR_DEFAULT_INTERFACE = '0.0.0.0'
47
BZR_DEFAULT_PORT = 4155
42
50
class _SmartStat(object):
83
88
one is being cloned from. Attributes such as the medium will
86
:param medium: The medium to use for this RemoteTransport. If None,
87
the medium from the _from_transport is shared. If both this
88
and _from_transport are None, a new medium will be built.
89
_from_transport and medium cannot both be specified.
91
:param medium: The medium to use for this RemoteTransport. This must be
92
supplied if _from_transport is None.
91
94
:param _client: Override the _SmartClient used by this transport. This
92
95
should only be used for testing purposes; normally this is
93
96
determined from the medium.
95
super(RemoteTransport, self).__init__(
96
url, _from_transport=_from_transport)
98
super(RemoteTransport, self).__init__(url,
99
_from_transport=_from_transport)
98
101
# The medium is the connection, except when we need to share it with
99
102
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
100
103
# what we want to share is really the shared connection.
102
if (_from_transport is not None
103
and isinstance(_from_transport, RemoteTransport)):
104
_client = _from_transport._client
105
elif _from_transport is None:
105
if _from_transport is None:
106
106
# If no _from_transport is specified, we need to intialize the
108
108
credentials = None
139
129
return None, None
141
def _report_activity(self, bytes, direction):
142
"""See Transport._report_activity.
144
Does nothing; the smart medium will report activity triggered by a
149
131
def is_readonly(self):
150
132
"""Smart server transport can do read/write file operations."""
152
resp = self._call2('Transport.is_readonly')
153
except errors.UnknownSmartMethod:
133
resp = self._call2('Transport.is_readonly')
134
if resp == ('yes', ):
136
elif resp == ('no', ):
138
elif (resp == ('error', "Generic bzr smart protocol error: "
139
"bad request 'Transport.is_readonly'") or
140
resp == ('error', "Generic bzr smart protocol error: "
141
"bad request u'Transport.is_readonly'")):
154
142
# XXX: nasty hack: servers before 0.16 don't have a
155
143
# 'Transport.is_readonly' verb, so we do what clients before 0.16
156
144
# did: assume False.
158
if resp == ('yes', ):
160
elif resp == ('no', ):
163
raise errors.UnexpectedSmartServerResponse(resp)
147
self._translate_error(resp)
148
raise errors.UnexpectedSmartServerResponse(resp)
165
150
def get_smart_client(self):
166
151
return self._get_connection()
168
153
def get_smart_medium(self):
169
154
return self._get_connection()
156
def get_shared_medium(self):
157
return self._get_shared_connection()
171
159
def _remote_path(self, relpath):
172
160
"""Returns the Unicode version of the absolute path for relpath."""
173
161
return self._combine_paths(self._path, relpath)
175
163
def _call(self, method, *args):
176
164
resp = self._call2(method, *args)
177
self._ensure_ok(resp)
165
self._translate_error(resp)
179
167
def _call2(self, method, *args):
180
168
"""Call a method on the remote server."""
182
return self._client.call(method, *args)
183
except errors.ErrorFromSmartServer, err:
184
# The first argument, if present, is always a path.
186
context = {'relpath': args[0]}
189
self._translate_error(err, **context)
169
return self._client.call(method, *args)
191
171
def _call_with_body_bytes(self, method, args, body):
192
172
"""Call a method on the remote server with body bytes."""
194
return self._client.call_with_body_bytes(method, args, body)
195
except errors.ErrorFromSmartServer, err:
196
# The first argument, if present, is always a path.
198
context = {'relpath': args[0]}
201
self._translate_error(err, **context)
173
return self._client.call_with_body_bytes(method, args, body)
203
175
def has(self, relpath):
204
176
"""Indicate whether a remote file of the given name exists or not.
211
183
elif resp == ('no', ):
214
raise errors.UnexpectedSmartServerResponse(resp)
186
self._translate_error(resp)
216
188
def get(self, relpath):
217
189
"""Return file-like object reading the contents of a remote file.
219
191
:see: Transport.get_bytes()/get_file()
221
193
return StringIO(self.get_bytes(relpath))
223
195
def get_bytes(self, relpath):
224
196
remote = self._remote_path(relpath)
226
resp, response_handler = self._client.call_expecting_body('get', remote)
227
except errors.ErrorFromSmartServer, err:
228
self._translate_error(err, relpath)
197
request = self.get_smart_medium().get_request()
198
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
199
smart_protocol.call('get', remote)
200
resp = smart_protocol.read_response_tuple(True)
229
201
if resp != ('ok', ):
230
response_handler.cancel_read_body()
231
raise errors.UnexpectedSmartServerResponse(resp)
232
return response_handler.read_body_bytes()
202
smart_protocol.cancel_read_body()
203
self._translate_error(resp, relpath)
204
return smart_protocol.read_body_bytes()
234
206
def _serialise_optional_mode(self, mode):
330
299
offsets = list(offsets)
332
301
sorted_offsets = sorted(offsets)
302
# turn the list of offsets into a stack
303
offset_stack = iter(offsets)
304
cur_offset_and_size = offset_stack.next()
333
305
coalesced = list(self._coalesce_offsets(sorted_offsets,
334
306
limit=self._max_readv_combine,
335
fudge_factor=self._bytes_to_read_before_seek,
336
max_size=self._max_readv_bytes))
338
# now that we've coallesced things, avoid making enormous requests
343
if c.length + cur_len > self._max_readv_bytes:
344
requests.append(cur_request)
348
cur_request.append(c)
351
requests.append(cur_request)
352
if 'hpss' in debug.debug_flags:
353
trace.mutter('%s.readv %s offsets => %s coalesced'
354
' => %s requests (%s)',
355
self.__class__.__name__, len(offsets), len(coalesced),
356
len(requests), sum(map(len, requests)))
307
fudge_factor=self._bytes_to_read_before_seek))
309
request = self.get_smart_medium().get_request()
310
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
311
smart_protocol.call_with_body_readv_array(
312
('readv', self._remote_path(relpath)),
313
[(c.start, c.length) for c in coalesced])
314
resp = smart_protocol.read_response_tuple(True)
316
if resp[0] != 'readv':
317
# This should raise an exception
318
smart_protocol.cancel_read_body()
319
self._translate_error(resp)
322
# FIXME: this should know how many bytes are needed, for clarity.
323
data = smart_protocol.read_body_bytes()
357
324
# Cache the results, but only until they have been fulfilled
359
# turn the list of offsets into a single stack to iterate
360
offset_stack = iter(offsets)
361
# using a list so it can be modified when passing down and coming back
362
next_offset = [offset_stack.next()]
363
for cur_request in requests:
365
result = self._client.call_with_body_readv_array(
366
('readv', self._remote_path(relpath),),
367
[(c.start, c.length) for c in cur_request])
368
resp, response_handler = result
369
except errors.ErrorFromSmartServer, err:
370
self._translate_error(err, relpath)
372
if resp[0] != 'readv':
373
# This should raise an exception
374
response_handler.cancel_read_body()
375
raise errors.UnexpectedSmartServerResponse(resp)
377
for res in self._handle_response(offset_stack, cur_request,
383
def _handle_response(self, offset_stack, coalesced, response_handler,
384
data_map, next_offset):
385
cur_offset_and_size = next_offset[0]
386
# FIXME: this should know how many bytes are needed, for clarity.
387
data = response_handler.read_body_bytes()
389
326
for c_offset in coalesced:
390
327
if len(data) < c_offset.length:
391
328
raise errors.ShortReadvError(relpath, c_offset.start,
392
329
c_offset.length, actual=len(data))
393
330
for suboffset, subsize in c_offset.ranges:
394
331
key = (c_offset.start+suboffset, subsize)
395
this_data = data[data_offset+suboffset:
396
data_offset+suboffset+subsize]
397
# Special case when the data is in-order, rather than packing
398
# into a map and then back out again. Benchmarking shows that
399
# this has 100% hit rate, but leave in the data_map work just
401
# TODO: Could we get away with using buffer() to avoid the
402
# memory copy? Callers would need to realize they may
403
# not have a real string.
404
if key == cur_offset_and_size:
405
yield cur_offset_and_size[0], this_data
406
cur_offset_and_size = next_offset[0] = offset_stack.next()
408
data_map[key] = this_data
409
data_offset += c_offset.length
332
data_map[key] = data[suboffset:suboffset+subsize]
333
data = data[c_offset.length:]
411
335
# Now that we've read some data, see if we can yield anything back
412
336
while cur_offset_and_size in data_map:
413
337
this_data = data_map.pop(cur_offset_and_size)
414
338
yield cur_offset_and_size[0], this_data
415
cur_offset_and_size = next_offset[0] = offset_stack.next()
339
cur_offset_and_size = offset_stack.next()
417
341
def rename(self, rel_from, rel_to):
418
342
self._call('rename',
427
351
def rmdir(self, relpath):
428
352
resp = self._call('rmdir', self._remote_path(relpath))
430
def _ensure_ok(self, resp):
432
raise errors.UnexpectedSmartServerResponse(resp)
434
def _translate_error(self, err, relpath=None):
435
remote._translate_error(err, path=relpath)
354
def _translate_error(self, resp, orig_path=None):
355
"""Raise an exception from a response"""
362
elif what == 'NoSuchFile':
363
if orig_path is not None:
364
error_path = orig_path
367
raise errors.NoSuchFile(error_path)
368
elif what == 'error':
369
raise errors.SmartProtocolError(unicode(resp[1]))
370
elif what == 'FileExists':
371
raise errors.FileExists(resp[1])
372
elif what == 'DirectoryNotEmpty':
373
raise errors.DirectoryNotEmpty(resp[1])
374
elif what == 'ShortReadvError':
375
raise errors.ShortReadvError(resp[1], int(resp[2]),
376
int(resp[3]), int(resp[4]))
377
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
378
encoding = str(resp[1]) # encoding must always be a string
382
reason = str(resp[5]) # reason must always be a string
383
if val.startswith('u:'):
384
val = val[2:].decode('utf-8')
385
elif val.startswith('s:'):
386
val = val[2:].decode('base64')
387
if what == 'UnicodeDecodeError':
388
raise UnicodeDecodeError(encoding, val, start, end, reason)
389
elif what == 'UnicodeEncodeError':
390
raise UnicodeEncodeError(encoding, val, start, end, reason)
391
elif what == "ReadOnlyError":
392
raise errors.TransportNotPossible('readonly transport')
393
elif what == "ReadError":
394
if orig_path is not None:
395
error_path = orig_path
398
raise errors.ReadError(error_path)
400
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
437
402
def disconnect(self):
438
403
self.get_smart_medium().disconnect()
405
def delete_tree(self, relpath):
406
raise errors.TransportNotPossible('readonly transport')
440
408
def stat(self, relpath):
441
409
resp = self._call2('stat', self._remote_path(relpath))
442
410
if resp[0] == 'stat':
443
411
return _SmartStat(int(resp[1]), int(resp[2], 8))
444
raise errors.UnexpectedSmartServerResponse(resp)
413
self._translate_error(resp)
446
415
## def lock_read(self, relpath):
447
416
## """Lock the given file for shared (read) access.
463
432
resp = self._call2('list_dir', self._remote_path(relpath))
464
433
if resp[0] == 'names':
465
434
return [name.encode('ascii') for name in resp[1:]]
466
raise errors.UnexpectedSmartServerResponse(resp)
436
self._translate_error(resp)
468
438
def iter_files_recursive(self):
469
439
resp = self._call2('iter_files_recursive', self._remote_path(''))
470
440
if resp[0] == 'names':
472
raise errors.UnexpectedSmartServerResponse(resp)
443
self._translate_error(resp)
475
446
class RemoteTCPTransport(RemoteTransport):
476
447
"""Connection to smart server over plain tcp.
478
449
This is essentially just a factory to get 'RemoteTransport(url,
479
450
SmartTCPClientMedium).
482
453
def _build_medium(self):
483
client_medium = medium.SmartTCPClientMedium(
484
self._host, self._port, self.base)
485
return client_medium, None
488
class RemoteTCPTransportV2Only(RemoteTransport):
489
"""Connection to smart server over plain tcp with the client hard-coded to
490
assume protocol v2 and remote server version <= 1.6.
492
This should only be used for testing.
495
def _build_medium(self):
496
client_medium = medium.SmartTCPClientMedium(
497
self._host, self._port, self.base)
498
client_medium._protocol_version = 2
499
client_medium._remember_remote_is_before((1, 6))
500
return client_medium, None
454
assert self.base.startswith('bzr://')
455
return medium.SmartTCPClientMedium(self._host, self._port), None
503
458
class RemoteSSHTransport(RemoteTransport):
510
465
def _build_medium(self):
466
assert self.base.startswith('bzr+ssh://')
467
# ssh will prompt the user for a password if needed and if none is
468
# provided but it will not give it back, so no credentials can be
511
470
location_config = config.LocationConfig(self.base)
512
471
bzr_remote_path = location_config.get_bzr_remote_path()
515
auth = config.AuthenticationConfig()
516
user = auth.get_user('ssh', self._host, self._port)
517
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
518
user, self._password, self.base,
519
bzr_remote_path=bzr_remote_path)
520
return client_medium, (user, self._password)
472
return medium.SmartSSHClientMedium(self._host, self._port,
473
self._user, self._password, bzr_remote_path=bzr_remote_path), None
523
476
class RemoteHTTPTransport(RemoteTransport):
524
477
"""Just a way to connect between a bzr+http:// url and http://.
526
479
This connection operates slightly differently than the RemoteSSHTransport.
527
480
It uses a plain http:// transport underneath, which defines what remote
528
481
.bzr/smart URL we are connected to. From there, all paths that are sent are
568
523
smart requests may be different). This is so that the server doesn't
569
524
have to handle .bzr/smart requests at arbitrary places inside .bzr
570
525
directories, just at the initial URL the user uses.
527
The exception is parent paths (i.e. relative_url of "..").
573
530
abs_url = self.abspath(relative_url)
575
532
abs_url = self.base
533
# We either use the exact same http_transport (for child locations), or
534
# a clone of the underlying http_transport (for parent locations). This
535
# means we share the connection.
536
norm_base = urlutils.normalize_url(self.base)
537
norm_abs_url = urlutils.normalize_url(abs_url)
538
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
539
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
540
http_transport = self._http_transport.clone(normalized_rel_url)
542
http_transport = self._http_transport
576
543
return RemoteHTTPTransport(abs_url,
577
544
_from_transport=self,
578
http_transport=self._http_transport)
580
def _redirected_to(self, source, target):
581
"""See transport._redirected_to"""
582
redirected = self._http_transport._redirected_to(source, target)
583
if (redirected is not None
584
and isinstance(redirected, type(self._http_transport))):
585
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
586
http_transport=redirected)
588
# Either None or a transport for a different protocol
592
class HintingSSHTransport(transport.Transport):
593
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
595
def __init__(self, url):
596
raise errors.UnsupportedProtocol(url,
597
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
545
http_transport=http_transport)
600
548
def get_test_permutations():
601
549
"""Return (transport, server) permutations for testing."""
602
550
### We may need a little more test framework support to construct an
603
551
### appropriate RemoteTransport in the future.
604
from bzrlib.tests import test_server
605
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
552
from bzrlib.smart import server
553
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]