55
58
The connection has a notion of the current directory to which it's
56
59
connected; this is incorporated in filenames passed to the server.
58
This supports some higher-level RPC operations and can also be treated
61
This supports some higher-level RPC operations and can also be treated
59
62
like a Transport to do file-like operations.
61
64
The connection can be made over a tcp socket, an ssh pipe or a series of
62
65
http requests. There are concrete subclasses for each type:
63
RemoteTCPTransport, etc.
66
SmartTCPTransport, etc.
66
# When making a readv request, cap it at requesting 5MB of data
67
_max_readv_bytes = 5*1024*1024
69
69
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
70
70
# responsibilities: Put those on SmartClient or similar. This is vital for
71
71
# the ability to support multiple versions of the smart protocol over time:
72
# RemoteTransport is an adapter from the Transport object model to the
72
# RemoteTransport is an adapter from the Transport object model to the
73
73
# SmartClient model, not an encoder.
75
# FIXME: the medium parameter should be private, only the tests requires
76
# it. It may be even clearer to define a TestRemoteTransport that handles
77
# the specific cases of providing a _client and/or a _medium, and leave
78
# RemoteTransport as an abstract class.
79
def __init__(self, url, _from_transport=None, medium=None, _client=None):
75
def __init__(self, url, clone_from=None, medium=None):
82
:param _from_transport: Another RemoteTransport instance that this
83
one is being cloned from. Attributes such as the medium will
86
:param medium: The medium to use for this RemoteTransport. If None,
87
the medium from the _from_transport is shared. If both this
88
and _from_transport are None, a new medium will be built.
89
_from_transport and medium cannot both be specified.
91
:param _client: Override the _SmartClient used by this transport. This
92
should only be used for testing purposes; normally this is
93
determined from the medium.
95
super(RemoteTransport, self).__init__(
96
url, _from_transport=_from_transport)
98
# The medium is the connection, except when we need to share it with
99
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
100
# what we want to share is really the shared connection.
102
if (_from_transport is not None
103
and isinstance(_from_transport, RemoteTransport)):
104
_client = _from_transport._client
105
elif _from_transport is None:
106
# If no _from_transport is specified, we need to intialize the
110
medium, credentials = self._build_medium()
111
if 'hpss' in debug.debug_flags:
112
trace.mutter('hpss: Built a new medium: %s',
113
medium.__class__.__name__)
114
self._shared_connection = transport._SharedConnection(medium,
118
# No medium was specified, so share the medium from the
120
medium = self._shared_connection.connection
122
raise AssertionError(
123
"Both _from_transport (%r) and medium (%r) passed to "
124
"RemoteTransport.__init__, but these parameters are mutally "
125
"exclusive." % (_from_transport, medium))
128
self._client = client._SmartClient(medium)
130
self._client = _client
132
def _build_medium(self):
133
"""Create the medium if _from_transport does not provide one.
135
The medium is analogous to the connection for ConnectedTransport: it
136
allows connection sharing.
141
def _report_activity(self, bytes, direction):
142
"""See Transport._report_activity.
144
Does nothing; the smart medium will report activity triggered by a
78
:param medium: The medium to use for this RemoteTransport. This must be
79
supplied if clone_from is None.
81
### Technically super() here is faulty because Transport's __init__
82
### fails to take 2 parameters, and if super were to choose a silly
83
### initialisation order things would blow up.
84
if not url.endswith('/'):
86
super(RemoteTransport, self).__init__(url)
87
self._scheme, self._username, self._password, self._host, self._port, self._path = \
88
transport.split_url(url)
89
if clone_from is None:
92
# credentials may be stripped from the base in some circumstances
93
# as yet to be clearly defined or documented, so copy them.
94
self._username = clone_from._username
95
# reuse same connection
96
self._medium = clone_from._medium
97
assert self._medium is not None
99
def abspath(self, relpath):
100
"""Return the full url to the given relative path.
102
@param relpath: the relative path or path components
103
@type relpath: str or list
105
return self._unparse_url(self._remote_path(relpath))
107
def clone(self, relative_url):
108
"""Make a new RemoteTransport related to me, sharing the same connection.
110
This essentially opens a handle on a different remote directory.
112
if relative_url is None:
113
return RemoteTransport(self.base, self)
115
return RemoteTransport(self.abspath(relative_url), self)
149
117
def is_readonly(self):
150
118
"""Smart server transport can do read/write file operations."""
152
resp = self._call2('Transport.is_readonly')
153
except errors.UnknownSmartMethod:
154
# XXX: nasty hack: servers before 0.16 don't have a
155
# 'Transport.is_readonly' verb, so we do what clients before 0.16
119
resp = self._call2('Transport.is_readonly')
158
120
if resp == ('yes', ):
160
122
elif resp == ('no', ):
163
raise errors.UnexpectedSmartServerResponse(resp)
125
self._translate_error(resp)
126
assert False, 'weird response %r' % (resp,)
165
128
def get_smart_client(self):
166
return self._get_connection()
168
131
def get_smart_medium(self):
169
return self._get_connection()
134
def _unparse_url(self, path):
135
"""Return URL for a path.
137
:see: SFTPUrlHandling._unparse_url
139
# TODO: Eventually it should be possible to unify this with
140
# SFTPUrlHandling._unparse_url?
143
path = urllib.quote(path)
144
netloc = urllib.quote(self._host)
145
if self._username is not None:
146
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
147
if self._port is not None:
148
netloc = '%s:%d' % (netloc, self._port)
149
return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))
171
151
def _remote_path(self, relpath):
172
152
"""Returns the Unicode version of the absolute path for relpath."""
175
155
def _call(self, method, *args):
176
156
resp = self._call2(method, *args)
177
self._ensure_ok(resp)
157
self._translate_error(resp)
179
159
def _call2(self, method, *args):
180
160
"""Call a method on the remote server."""
182
return self._client.call(method, *args)
183
except errors.ErrorFromSmartServer, err:
184
# The first argument, if present, is always a path.
186
context = {'relpath': args[0]}
189
self._translate_error(err, **context)
161
return client.SmartClient(self._medium).call(method, *args)
191
163
def _call_with_body_bytes(self, method, args, body):
192
164
"""Call a method on the remote server with body bytes."""
194
return self._client.call_with_body_bytes(method, args, body)
195
except errors.ErrorFromSmartServer, err:
196
# The first argument, if present, is always a path.
198
context = {'relpath': args[0]}
201
self._translate_error(err, **context)
165
smart_client = client.SmartClient(self._medium)
166
return smart_client.call_with_body_bytes(method, args, body)
203
168
def has(self, relpath):
204
169
"""Indicate whether a remote file of the given name exists or not.
211
176
elif resp == ('no', ):
214
raise errors.UnexpectedSmartServerResponse(resp)
179
self._translate_error(resp)
216
181
def get(self, relpath):
217
182
"""Return file-like object reading the contents of a remote file.
219
184
:see: Transport.get_bytes()/get_file()
221
186
return StringIO(self.get_bytes(relpath))
223
188
def get_bytes(self, relpath):
224
189
remote = self._remote_path(relpath)
226
resp, response_handler = self._client.call_expecting_body('get', remote)
227
except errors.ErrorFromSmartServer, err:
228
self._translate_error(err, relpath)
190
request = self._medium.get_request()
191
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
192
smart_protocol.call('get', remote)
193
resp = smart_protocol.read_response_tuple(True)
229
194
if resp != ('ok', ):
230
response_handler.cancel_read_body()
231
raise errors.UnexpectedSmartServerResponse(resp)
232
return response_handler.read_body_bytes()
195
smart_protocol.cancel_read_body()
196
self._translate_error(resp, relpath)
197
return smart_protocol.read_body_bytes()
234
199
def _serialise_optional_mode(self, mode):
309
267
if resp[0] == 'appended':
310
268
return int(resp[1])
311
raise errors.UnexpectedSmartServerResponse(resp)
269
self._translate_error(resp)
313
271
def delete(self, relpath):
314
272
resp = self._call2('delete', self._remote_path(relpath))
315
self._ensure_ok(resp)
317
def external_url(self):
318
"""See bzrlib.transport.Transport.external_url."""
319
# the external path for RemoteTransports is the base
322
def recommended_page_size(self):
323
"""Return the recommended page size for this transport."""
326
def _readv(self, relpath, offsets):
273
self._translate_error(resp)
275
def readv(self, relpath, offsets):
330
279
offsets = list(offsets)
332
281
sorted_offsets = sorted(offsets)
282
# turn the list of offsets into a stack
283
offset_stack = iter(offsets)
284
cur_offset_and_size = offset_stack.next()
333
285
coalesced = list(self._coalesce_offsets(sorted_offsets,
334
286
limit=self._max_readv_combine,
335
fudge_factor=self._bytes_to_read_before_seek,
336
max_size=self._max_readv_bytes))
338
# now that we've coallesced things, avoid making enormous requests
343
if c.length + cur_len > self._max_readv_bytes:
344
requests.append(cur_request)
348
cur_request.append(c)
351
requests.append(cur_request)
352
if 'hpss' in debug.debug_flags:
353
trace.mutter('%s.readv %s offsets => %s coalesced'
354
' => %s requests (%s)',
355
self.__class__.__name__, len(offsets), len(coalesced),
356
len(requests), sum(map(len, requests)))
287
fudge_factor=self._bytes_to_read_before_seek))
289
request = self._medium.get_request()
290
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
291
smart_protocol.call_with_body_readv_array(
292
('readv', self._remote_path(relpath)),
293
[(c.start, c.length) for c in coalesced])
294
resp = smart_protocol.read_response_tuple(True)
296
if resp[0] != 'readv':
297
# This should raise an exception
298
smart_protocol.cancel_read_body()
299
self._translate_error(resp)
302
# FIXME: this should know how many bytes are needed, for clarity.
303
data = smart_protocol.read_body_bytes()
357
304
# Cache the results, but only until they have been fulfilled
359
# turn the list of offsets into a single stack to iterate
360
offset_stack = iter(offsets)
361
# using a list so it can be modified when passing down and coming back
362
next_offset = [offset_stack.next()]
363
for cur_request in requests:
365
result = self._client.call_with_body_readv_array(
366
('readv', self._remote_path(relpath),),
367
[(c.start, c.length) for c in cur_request])
368
resp, response_handler = result
369
except errors.ErrorFromSmartServer, err:
370
self._translate_error(err, relpath)
372
if resp[0] != 'readv':
373
# This should raise an exception
374
response_handler.cancel_read_body()
375
raise errors.UnexpectedSmartServerResponse(resp)
377
for res in self._handle_response(offset_stack, cur_request,
383
def _handle_response(self, offset_stack, coalesced, response_handler,
384
data_map, next_offset):
385
cur_offset_and_size = next_offset[0]
386
# FIXME: this should know how many bytes are needed, for clarity.
387
data = response_handler.read_body_bytes()
389
306
for c_offset in coalesced:
390
307
if len(data) < c_offset.length:
391
308
raise errors.ShortReadvError(relpath, c_offset.start,
392
309
c_offset.length, actual=len(data))
393
310
for suboffset, subsize in c_offset.ranges:
394
311
key = (c_offset.start+suboffset, subsize)
395
this_data = data[data_offset+suboffset:
396
data_offset+suboffset+subsize]
397
# Special case when the data is in-order, rather than packing
398
# into a map and then back out again. Benchmarking shows that
399
# this has 100% hit rate, but leave in the data_map work just
401
# TODO: Could we get away with using buffer() to avoid the
402
# memory copy? Callers would need to realize they may
403
# not have a real string.
404
if key == cur_offset_and_size:
405
yield cur_offset_and_size[0], this_data
406
cur_offset_and_size = next_offset[0] = offset_stack.next()
408
data_map[key] = this_data
409
data_offset += c_offset.length
312
data_map[key] = data[suboffset:suboffset+subsize]
313
data = data[c_offset.length:]
411
315
# Now that we've read some data, see if we can yield anything back
412
316
while cur_offset_and_size in data_map:
413
317
this_data = data_map.pop(cur_offset_and_size)
414
318
yield cur_offset_and_size[0], this_data
415
cur_offset_and_size = next_offset[0] = offset_stack.next()
319
cur_offset_and_size = offset_stack.next()
417
321
def rename(self, rel_from, rel_to):
418
322
self._call('rename',
427
331
def rmdir(self, relpath):
428
332
resp = self._call('rmdir', self._remote_path(relpath))
430
def _ensure_ok(self, resp):
432
raise errors.UnexpectedSmartServerResponse(resp)
434
def _translate_error(self, err, relpath=None):
435
remote._translate_error(err, path=relpath)
334
def _translate_error(self, resp, orig_path=None):
335
"""Raise an exception from a response"""
342
elif what == 'NoSuchFile':
343
if orig_path is not None:
344
error_path = orig_path
347
raise errors.NoSuchFile(error_path)
348
elif what == 'error':
349
raise errors.SmartProtocolError(unicode(resp[1]))
350
elif what == 'FileExists':
351
raise errors.FileExists(resp[1])
352
elif what == 'DirectoryNotEmpty':
353
raise errors.DirectoryNotEmpty(resp[1])
354
elif what == 'ShortReadvError':
355
raise errors.ShortReadvError(resp[1], int(resp[2]),
356
int(resp[3]), int(resp[4]))
357
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
358
encoding = str(resp[1]) # encoding must always be a string
362
reason = str(resp[5]) # reason must always be a string
363
if val.startswith('u:'):
364
val = val[2:].decode('utf-8')
365
elif val.startswith('s:'):
366
val = val[2:].decode('base64')
367
if what == 'UnicodeDecodeError':
368
raise UnicodeDecodeError(encoding, val, start, end, reason)
369
elif what == 'UnicodeEncodeError':
370
raise UnicodeEncodeError(encoding, val, start, end, reason)
371
elif what == "ReadOnlyError":
372
raise errors.TransportNotPossible('readonly transport')
374
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
437
376
def disconnect(self):
438
self.get_smart_medium().disconnect()
377
self._medium.disconnect()
379
def delete_tree(self, relpath):
380
raise errors.TransportNotPossible('readonly transport')
440
382
def stat(self, relpath):
441
383
resp = self._call2('stat', self._remote_path(relpath))
442
384
if resp[0] == 'stat':
443
385
return _SmartStat(int(resp[1]), int(resp[2], 8))
444
raise errors.UnexpectedSmartServerResponse(resp)
387
self._translate_error(resp)
446
389
## def lock_read(self, relpath):
447
390
## """Lock the given file for shared (read) access.
463
406
resp = self._call2('list_dir', self._remote_path(relpath))
464
407
if resp[0] == 'names':
465
408
return [name.encode('ascii') for name in resp[1:]]
466
raise errors.UnexpectedSmartServerResponse(resp)
410
self._translate_error(resp)
468
412
def iter_files_recursive(self):
469
413
resp = self._call2('iter_files_recursive', self._remote_path(''))
470
414
if resp[0] == 'names':
472
raise errors.UnexpectedSmartServerResponse(resp)
475
class RemoteTCPTransport(RemoteTransport):
417
self._translate_error(resp)
420
class SmartTCPTransport(RemoteTransport):
476
421
"""Connection to smart server over plain tcp.
478
423
This is essentially just a factory to get 'RemoteTransport(url,
479
424
SmartTCPClientMedium).
482
def _build_medium(self):
483
client_medium = medium.SmartTCPClientMedium(
484
self._host, self._port, self.base)
485
return client_medium, None
488
class RemoteTCPTransportV2Only(RemoteTransport):
489
"""Connection to smart server over plain tcp with the client hard-coded to
490
assume protocol v2 and remote server version <= 1.6.
492
This should only be used for testing.
495
def _build_medium(self):
496
client_medium = medium.SmartTCPClientMedium(
497
self._host, self._port, self.base)
498
client_medium._protocol_version = 2
499
client_medium._remember_remote_is_before((1, 6))
500
return client_medium, None
503
class RemoteSSHTransport(RemoteTransport):
427
def __init__(self, url):
428
_scheme, _username, _password, _host, _port, _path = \
429
transport.split_url(url)
431
_port = BZR_DEFAULT_PORT
435
except (ValueError, TypeError), e:
436
raise errors.InvalidURL(
437
path=url, extra="invalid port %s" % _port)
438
client_medium = medium.SmartTCPClientMedium(_host, _port)
439
super(SmartTCPTransport, self).__init__(url, medium=client_medium)
442
class SmartSSHTransport(RemoteTransport):
504
443
"""Connection to smart server over SSH.
506
445
This is essentially just a factory to get 'RemoteTransport(url,
507
446
SmartSSHClientMedium).
510
def _build_medium(self):
511
location_config = config.LocationConfig(self.base)
512
bzr_remote_path = location_config.get_bzr_remote_path()
515
auth = config.AuthenticationConfig()
516
user = auth.get_user('ssh', self._host, self._port)
517
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
518
user, self._password, self.base,
519
bzr_remote_path=bzr_remote_path)
520
return client_medium, (user, self._password)
523
class RemoteHTTPTransport(RemoteTransport):
449
def __init__(self, url):
450
_scheme, _username, _password, _host, _port, _path = \
451
transport.split_url(url)
453
if _port is not None:
455
except (ValueError, TypeError), e:
456
raise errors.InvalidURL(path=url, extra="invalid port %s" %
458
client_medium = medium.SmartSSHClientMedium(_host, _port,
459
_username, _password)
460
super(SmartSSHTransport, self).__init__(url, medium=client_medium)
463
class SmartHTTPTransport(RemoteTransport):
524
464
"""Just a way to connect between a bzr+http:// url and http://.
526
This connection operates slightly differently than the RemoteSSHTransport.
466
This connection operates slightly differently than the SmartSSHTransport.
527
467
It uses a plain http:// transport underneath, which defines what remote
528
468
.bzr/smart URL we are connected to. From there, all paths that are sent are
529
469
sent as relative paths, this way, the remote side can properly
531
471
HTTP path into a local path.
534
def __init__(self, base, _from_transport=None, http_transport=None):
474
def __init__(self, url, http_transport=None):
475
assert url.startswith('bzr+http://')
535
477
if http_transport is None:
536
# FIXME: the password may be lost here because it appears in the
537
# url only for an intial construction (when the url came from the
539
http_url = base[len('bzr+'):]
478
http_url = url[len('bzr+'):]
540
479
self._http_transport = transport.get_transport(http_url)
542
481
self._http_transport = http_transport
543
super(RemoteHTTPTransport, self).__init__(
544
base, _from_transport=_from_transport)
546
def _build_medium(self):
547
# We let http_transport take care of the credentials
548
return self._http_transport.get_smart_medium(), None
482
http_medium = self._http_transport.get_smart_medium()
483
super(SmartHTTPTransport, self).__init__(url, medium=http_medium)
550
485
def _remote_path(self, relpath):
551
"""After connecting, HTTP Transport only deals in relative URLs."""
552
# Adjust the relpath based on which URL this smart transport is
554
http_base = urlutils.normalize_url(self.get_smart_medium().base)
555
url = urlutils.join(self.base[len('bzr+'):], relpath)
556
url = urlutils.normalize_url(url)
557
return urlutils.relative_url(http_base, url)
486
"""After connecting HTTP Transport only deals in relative URLs."""
492
def abspath(self, relpath):
493
"""Return the full url to the given relative path.
495
:param relpath: the relative path or path components
496
:type relpath: str or list
498
return self._unparse_url(self._combine_paths(self._path, relpath))
559
500
def clone(self, relative_url):
560
"""Make a new RemoteHTTPTransport related to me.
501
"""Make a new SmartHTTPTransport related to me.
562
503
This is re-implemented rather than using the default
563
RemoteTransport.clone() because we must be careful about the underlying
504
SmartTransport.clone() because we must be careful about the underlying
566
Also, the cloned smart transport will POST to the same .bzr/smart
567
location as this transport (although obviously the relative paths in the
568
smart requests may be different). This is so that the server doesn't
569
have to handle .bzr/smart requests at arbitrary places inside .bzr
570
directories, just at the initial URL the user uses.
573
508
abs_url = self.abspath(relative_url)
575
510
abs_url = self.base
576
return RemoteHTTPTransport(abs_url,
577
_from_transport=self,
578
http_transport=self._http_transport)
580
def _redirected_to(self, source, target):
581
"""See transport._redirected_to"""
582
redirected = self._http_transport._redirected_to(source, target)
583
if (redirected is not None
584
and isinstance(redirected, type(self._http_transport))):
585
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
586
http_transport=redirected)
588
# Either None or a transport for a different protocol
592
class HintingSSHTransport(transport.Transport):
593
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
595
def __init__(self, url):
596
raise errors.UnsupportedProtocol(url,
597
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
511
# By cloning the underlying http_transport, we are able to share the
513
new_transport = self._http_transport.clone(relative_url)
514
return SmartHTTPTransport(abs_url, http_transport=new_transport)
600
517
def get_test_permutations():
601
518
"""Return (transport, server) permutations for testing."""
602
519
### We may need a little more test framework support to construct an
603
520
### appropriate RemoteTransport in the future.
604
from bzrlib.tests import test_server
605
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
521
from bzrlib.smart import server
522
return [(SmartTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
