1
# Copyright (C) 2006-2012, 2016 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from breezy.bzr.smart.
23
from __future__ import absolute_import
25
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
38
from ..sixish import (
41
from ..bzr.smart import client, medium
44
class _SmartStat(object):
46
def __init__(self, size, mode):
51
class RemoteTransport(transport.ConnectedTransport):
52
"""Connection to a smart server.
54
The connection holds references to the medium that can be used to send
55
requests to the server.
57
The connection has a notion of the current directory to which it's
58
connected; this is incorporated in filenames passed to the server.
60
This supports some higher-level RPC operations and can also be treated
61
like a Transport to do file-like operations.
63
The connection can be made over a tcp socket, an ssh pipe or a series of
64
http requests. There are concrete subclasses for each type:
65
RemoteTCPTransport, etc.
68
# When making a readv request, cap it at requesting 5MB of data
69
_max_readv_bytes = 5*1024*1024
71
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
72
# responsibilities: Put those on SmartClient or similar. This is vital for
73
# the ability to support multiple versions of the smart protocol over time:
74
# RemoteTransport is an adapter from the Transport object model to the
75
# SmartClient model, not an encoder.
77
# FIXME: the medium parameter should be private, only the tests requires
78
# it. It may be even clearer to define a TestRemoteTransport that handles
79
# the specific cases of providing a _client and/or a _medium, and leave
80
# RemoteTransport as an abstract class.
81
def __init__(self, url, _from_transport=None, medium=None, _client=None):
84
:param _from_transport: Another RemoteTransport instance that this
85
one is being cloned from. Attributes such as the medium will
88
:param medium: The medium to use for this RemoteTransport. If None,
89
the medium from the _from_transport is shared. If both this
90
and _from_transport are None, a new medium will be built.
91
_from_transport and medium cannot both be specified.
93
:param _client: Override the _SmartClient used by this transport. This
94
should only be used for testing purposes; normally this is
95
determined from the medium.
97
super(RemoteTransport, self).__init__(
98
url, _from_transport=_from_transport)
100
# The medium is the connection, except when we need to share it with
101
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
102
# what we want to share is really the shared connection.
104
if (_from_transport is not None
105
and isinstance(_from_transport, RemoteTransport)):
106
_client = _from_transport._client
107
elif _from_transport is None:
108
# If no _from_transport is specified, we need to intialize the
112
medium, credentials = self._build_medium()
113
if 'hpss' in debug.debug_flags:
114
trace.mutter('hpss: Built a new medium: %s',
115
medium.__class__.__name__)
116
self._shared_connection = transport._SharedConnection(medium,
120
# No medium was specified, so share the medium from the
122
medium = self._shared_connection.connection
124
raise AssertionError(
125
"Both _from_transport (%r) and medium (%r) passed to "
126
"RemoteTransport.__init__, but these parameters are mutally "
127
"exclusive." % (_from_transport, medium))
130
self._client = client._SmartClient(medium)
132
self._client = _client
134
def _build_medium(self):
135
"""Create the medium if _from_transport does not provide one.
137
The medium is analogous to the connection for ConnectedTransport: it
138
allows connection sharing.
143
def _report_activity(self, bytes, direction):
144
"""See Transport._report_activity.
146
Does nothing; the smart medium will report activity triggered by a
151
def is_readonly(self):
152
"""Smart server transport can do read/write file operations."""
154
resp = self._call2('Transport.is_readonly')
155
except errors.UnknownSmartMethod:
156
# XXX: nasty hack: servers before 0.16 don't have a
157
# 'Transport.is_readonly' verb, so we do what clients before 0.16
160
if resp == ('yes', ):
162
elif resp == ('no', ):
165
raise errors.UnexpectedSmartServerResponse(resp)
167
def get_smart_client(self):
168
return self._get_connection()
170
def get_smart_medium(self):
171
return self._get_connection()
173
def _remote_path(self, relpath):
174
"""Returns the Unicode version of the absolute path for relpath."""
175
return urlutils.URL._combine_paths(self._parsed_url.path, relpath)
177
def _call(self, method, *args):
178
resp = self._call2(method, *args)
179
self._ensure_ok(resp)
181
def _call2(self, method, *args):
182
"""Call a method on the remote server."""
184
return self._client.call(method, *args)
185
except errors.ErrorFromSmartServer as err:
186
# The first argument, if present, is always a path.
188
context = {'relpath': args[0]}
191
self._translate_error(err, **context)
193
def _call_with_body_bytes(self, method, args, body):
194
"""Call a method on the remote server with body bytes."""
196
return self._client.call_with_body_bytes(method, args, body)
197
except errors.ErrorFromSmartServer as err:
198
# The first argument, if present, is always a path.
200
context = {'relpath': args[0]}
203
self._translate_error(err, **context)
205
def has(self, relpath):
206
"""Indicate whether a remote file of the given name exists or not.
208
:see: Transport.has()
210
resp = self._call2('has', self._remote_path(relpath))
211
if resp == ('yes', ):
213
elif resp == ('no', ):
216
raise errors.UnexpectedSmartServerResponse(resp)
218
def get(self, relpath):
219
"""Return file-like object reading the contents of a remote file.
221
:see: Transport.get_bytes()/get_file()
223
return BytesIO(self.get_bytes(relpath))
225
def get_bytes(self, relpath):
226
remote = self._remote_path(relpath)
228
resp, response_handler = self._client.call_expecting_body('get', remote)
229
except errors.ErrorFromSmartServer as err:
230
self._translate_error(err, relpath)
232
response_handler.cancel_read_body()
233
raise errors.UnexpectedSmartServerResponse(resp)
234
return response_handler.read_body_bytes()
236
def _serialise_optional_mode(self, mode):
242
def mkdir(self, relpath, mode=None):
243
resp = self._call2('mkdir', self._remote_path(relpath),
244
self._serialise_optional_mode(mode))
246
def open_write_stream(self, relpath, mode=None):
247
"""See Transport.open_write_stream."""
248
self.put_bytes(relpath, "", mode)
249
result = transport.AppendBasedFileStream(self, relpath)
250
transport._file_streams[self.abspath(relpath)] = result
253
def put_bytes(self, relpath, raw_bytes, mode=None):
254
if not isinstance(raw_bytes, str):
256
'raw_bytes must be a plain string, not %s' % type(raw_bytes))
257
resp = self._call_with_body_bytes(
259
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
261
self._ensure_ok(resp)
262
return len(raw_bytes)
264
def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None,
265
create_parent_dir=False,
267
"""See Transport.put_bytes_non_atomic."""
268
# FIXME: no encoding in the transport!
269
create_parent_str = 'F'
270
if create_parent_dir:
271
create_parent_str = 'T'
273
resp = self._call_with_body_bytes(
275
(self._remote_path(relpath), self._serialise_optional_mode(mode),
276
create_parent_str, self._serialise_optional_mode(dir_mode)),
278
self._ensure_ok(resp)
280
def put_file(self, relpath, upload_file, mode=None):
281
# its not ideal to seek back, but currently put_non_atomic_file depends
282
# on transports not reading before failing - which is a faulty
283
# assumption I think - RBC 20060915
284
pos = upload_file.tell()
286
return self.put_bytes(relpath, upload_file.read(), mode)
288
upload_file.seek(pos)
291
def put_file_non_atomic(self, relpath, f, mode=None,
292
create_parent_dir=False,
294
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
295
create_parent_dir=create_parent_dir,
298
def append_file(self, relpath, from_file, mode=None):
299
return self.append_bytes(relpath, from_file.read(), mode)
301
def append_bytes(self, relpath, bytes, mode=None):
302
resp = self._call_with_body_bytes(
304
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
306
if resp[0] == 'appended':
308
raise errors.UnexpectedSmartServerResponse(resp)
310
def delete(self, relpath):
311
resp = self._call2('delete', self._remote_path(relpath))
312
self._ensure_ok(resp)
314
def external_url(self):
315
"""See breezy.transport.Transport.external_url."""
316
# the external path for RemoteTransports is the base
319
def recommended_page_size(self):
320
"""Return the recommended page size for this transport."""
323
def _readv(self, relpath, offsets):
327
offsets = list(offsets)
329
sorted_offsets = sorted(offsets)
330
coalesced = list(self._coalesce_offsets(sorted_offsets,
331
limit=self._max_readv_combine,
332
fudge_factor=self._bytes_to_read_before_seek,
333
max_size=self._max_readv_bytes))
335
# now that we've coallesced things, avoid making enormous requests
340
if c.length + cur_len > self._max_readv_bytes:
341
requests.append(cur_request)
345
cur_request.append(c)
348
requests.append(cur_request)
349
if 'hpss' in debug.debug_flags:
350
trace.mutter('%s.readv %s offsets => %s coalesced'
351
' => %s requests (%s)',
352
self.__class__.__name__, len(offsets), len(coalesced),
353
len(requests), sum(map(len, requests)))
354
# Cache the results, but only until they have been fulfilled
356
# turn the list of offsets into a single stack to iterate
357
offset_stack = iter(offsets)
358
# using a list so it can be modified when passing down and coming back
359
next_offset = [next(offset_stack)]
360
for cur_request in requests:
362
result = self._client.call_with_body_readv_array(
363
('readv', self._remote_path(relpath),),
364
[(c.start, c.length) for c in cur_request])
365
resp, response_handler = result
366
except errors.ErrorFromSmartServer as err:
367
self._translate_error(err, relpath)
369
if resp[0] != 'readv':
370
# This should raise an exception
371
response_handler.cancel_read_body()
372
raise errors.UnexpectedSmartServerResponse(resp)
374
for res in self._handle_response(offset_stack, cur_request,
380
def _handle_response(self, offset_stack, coalesced, response_handler,
381
data_map, next_offset):
382
cur_offset_and_size = next_offset[0]
383
# FIXME: this should know how many bytes are needed, for clarity.
384
data = response_handler.read_body_bytes()
386
for c_offset in coalesced:
387
if len(data) < c_offset.length:
388
raise errors.ShortReadvError(relpath, c_offset.start,
389
c_offset.length, actual=len(data))
390
for suboffset, subsize in c_offset.ranges:
391
key = (c_offset.start+suboffset, subsize)
392
this_data = data[data_offset+suboffset:
393
data_offset+suboffset+subsize]
394
# Special case when the data is in-order, rather than packing
395
# into a map and then back out again. Benchmarking shows that
396
# this has 100% hit rate, but leave in the data_map work just
398
# TODO: Could we get away with using buffer() to avoid the
399
# memory copy? Callers would need to realize they may
400
# not have a real string.
401
if key == cur_offset_and_size:
402
yield cur_offset_and_size[0], this_data
403
cur_offset_and_size = next_offset[0] = next(offset_stack)
405
data_map[key] = this_data
406
data_offset += c_offset.length
408
# Now that we've read some data, see if we can yield anything back
409
while cur_offset_and_size in data_map:
410
this_data = data_map.pop(cur_offset_and_size)
411
yield cur_offset_and_size[0], this_data
412
cur_offset_and_size = next_offset[0] = next(offset_stack)
414
def rename(self, rel_from, rel_to):
416
self._remote_path(rel_from),
417
self._remote_path(rel_to))
419
def move(self, rel_from, rel_to):
421
self._remote_path(rel_from),
422
self._remote_path(rel_to))
424
def rmdir(self, relpath):
425
resp = self._call('rmdir', self._remote_path(relpath))
427
def _ensure_ok(self, resp):
429
raise errors.UnexpectedSmartServerResponse(resp)
431
def _translate_error(self, err, relpath=None):
432
remote._translate_error(err, path=relpath)
434
def disconnect(self):
435
m = self.get_smart_medium()
439
def stat(self, relpath):
440
resp = self._call2('stat', self._remote_path(relpath))
441
if resp[0] == 'stat':
442
return _SmartStat(int(resp[1]), int(resp[2], 8))
443
raise errors.UnexpectedSmartServerResponse(resp)
445
## def lock_read(self, relpath):
446
## """Lock the given file for shared (read) access.
447
## :return: A lock object, which should be passed to Transport.unlock()
449
## # The old RemoteBranch ignore lock for reading, so we will
450
## # continue that tradition and return a bogus lock object.
451
## class BogusLock(object):
452
## def __init__(self, path):
456
## return BogusLock(relpath)
461
def list_dir(self, relpath):
462
resp = self._call2('list_dir', self._remote_path(relpath))
463
if resp[0] == 'names':
464
return [name.encode('ascii') for name in resp[1:]]
465
raise errors.UnexpectedSmartServerResponse(resp)
467
def iter_files_recursive(self):
468
resp = self._call2('iter_files_recursive', self._remote_path(''))
469
if resp[0] == 'names':
471
raise errors.UnexpectedSmartServerResponse(resp)
474
class RemoteTCPTransport(RemoteTransport):
475
"""Connection to smart server over plain tcp.
477
This is essentially just a factory to get 'RemoteTransport(url,
478
SmartTCPClientMedium).
481
def _build_medium(self):
482
client_medium = medium.SmartTCPClientMedium(
483
self._parsed_url.host, self._parsed_url.port, self.base)
484
return client_medium, None
487
class RemoteTCPTransportV2Only(RemoteTransport):
488
"""Connection to smart server over plain tcp with the client hard-coded to
489
assume protocol v2 and remote server version <= 1.6.
491
This should only be used for testing.
494
def _build_medium(self):
495
client_medium = medium.SmartTCPClientMedium(
496
self._parsed_url.host, self._parsed_url.port, self.base)
497
client_medium._protocol_version = 2
498
client_medium._remember_remote_is_before((1, 6))
499
return client_medium, None
502
class RemoteSSHTransport(RemoteTransport):
503
"""Connection to smart server over SSH.
505
This is essentially just a factory to get 'RemoteTransport(url,
506
SmartSSHClientMedium).
509
def _build_medium(self):
510
location_config = config.LocationConfig(self.base)
511
bzr_remote_path = location_config.get_bzr_remote_path()
512
user = self._parsed_url.user
514
auth = config.AuthenticationConfig()
515
user = auth.get_user('ssh', self._parsed_url.host,
516
self._parsed_url.port)
517
ssh_params = medium.SSHParams(self._parsed_url.host,
518
self._parsed_url.port, user, self._parsed_url.password,
520
client_medium = medium.SmartSSHClientMedium(self.base, ssh_params)
521
return client_medium, (user, self._parsed_url.password)
524
class RemoteHTTPTransport(RemoteTransport):
525
"""Just a way to connect between a bzr+http:// url and http://.
527
This connection operates slightly differently than the RemoteSSHTransport.
528
It uses a plain http:// transport underneath, which defines what remote
529
.bzr/smart URL we are connected to. From there, all paths that are sent are
530
sent as relative paths, this way, the remote side can properly
531
de-reference them, since it is likely doing rewrite rules to translate an
532
HTTP path into a local path.
535
def __init__(self, base, _from_transport=None, http_transport=None):
536
if http_transport is None:
537
# FIXME: the password may be lost here because it appears in the
538
# url only for an intial construction (when the url came from the
540
http_url = base[len('bzr+'):]
541
self._http_transport = transport.get_transport_from_url(http_url)
543
self._http_transport = http_transport
544
super(RemoteHTTPTransport, self).__init__(
545
base, _from_transport=_from_transport)
547
def _build_medium(self):
548
# We let http_transport take care of the credentials
549
return self._http_transport.get_smart_medium(), None
551
def _remote_path(self, relpath):
552
"""After connecting, HTTP Transport only deals in relative URLs."""
553
# Adjust the relpath based on which URL this smart transport is
555
http_base = urlutils.normalize_url(self.get_smart_medium().base)
556
url = urlutils.join(self.base[len('bzr+'):], relpath)
557
url = urlutils.normalize_url(url)
558
return urlutils.relative_url(http_base, url)
560
def clone(self, relative_url):
561
"""Make a new RemoteHTTPTransport related to me.
563
This is re-implemented rather than using the default
564
RemoteTransport.clone() because we must be careful about the underlying
567
Also, the cloned smart transport will POST to the same .bzr/smart
568
location as this transport (although obviously the relative paths in the
569
smart requests may be different). This is so that the server doesn't
570
have to handle .bzr/smart requests at arbitrary places inside .bzr
571
directories, just at the initial URL the user uses.
574
abs_url = self.abspath(relative_url)
577
return RemoteHTTPTransport(abs_url,
578
_from_transport=self,
579
http_transport=self._http_transport)
581
def _redirected_to(self, source, target):
582
"""See transport._redirected_to"""
583
redirected = self._http_transport._redirected_to(source, target)
584
if (redirected is not None
585
and isinstance(redirected, type(self._http_transport))):
586
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
587
http_transport=redirected)
589
# Either None or a transport for a different protocol
593
class HintingSSHTransport(transport.Transport):
594
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
596
def __init__(self, url):
597
raise errors.UnsupportedProtocol(url,
598
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
601
def get_test_permutations():
602
"""Return (transport, server) permutations for testing."""
603
### We may need a little more test framework support to construct an
604
### appropriate RemoteTransport in the future.
605
from ..tests import test_server
606
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
added
removed
breezy/transport/remote.py
