1
# Copyright (C) 2006-2012, 2016 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from breezy.bzr.smart.
23
from __future__ import absolute_import
25
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
27
from io import BytesIO
40
from ..sixish import PY3
41
from ..bzr.smart import client, medium
44
class _SmartStat(object):
46
def __init__(self, size, mode):
51
class RemoteTransport(transport.ConnectedTransport):
52
"""Connection to a smart server.
54
The connection holds references to the medium that can be used to send
55
requests to the server.
57
The connection has a notion of the current directory to which it's
58
connected; this is incorporated in filenames passed to the server.
60
This supports some higher-level RPC operations and can also be treated
61
like a Transport to do file-like operations.
63
The connection can be made over a tcp socket, an ssh pipe or a series of
64
http requests. There are concrete subclasses for each type:
65
RemoteTCPTransport, etc.
68
# When making a readv request, cap it at requesting 5MB of data
69
_max_readv_bytes = 5 * 1024 * 1024
71
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
72
# responsibilities: Put those on SmartClient or similar. This is vital for
73
# the ability to support multiple versions of the smart protocol over time:
74
# RemoteTransport is an adapter from the Transport object model to the
75
# SmartClient model, not an encoder.
77
# FIXME: the medium parameter should be private, only the tests requires
78
# it. It may be even clearer to define a TestRemoteTransport that handles
79
# the specific cases of providing a _client and/or a _medium, and leave
80
# RemoteTransport as an abstract class.
81
def __init__(self, url, _from_transport=None, medium=None, _client=None):
84
:param _from_transport: Another RemoteTransport instance that this
85
one is being cloned from. Attributes such as the medium will
88
:param medium: The medium to use for this RemoteTransport. If None,
89
the medium from the _from_transport is shared. If both this
90
and _from_transport are None, a new medium will be built.
91
_from_transport and medium cannot both be specified.
93
:param _client: Override the _SmartClient used by this transport. This
94
should only be used for testing purposes; normally this is
95
determined from the medium.
97
super(RemoteTransport, self).__init__(
98
url, _from_transport=_from_transport)
100
# The medium is the connection, except when we need to share it with
101
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
102
# what we want to share is really the shared connection.
104
if (_from_transport is not None
105
and isinstance(_from_transport, RemoteTransport)):
106
_client = _from_transport._client
107
elif _from_transport is None:
108
# If no _from_transport is specified, we need to intialize the
112
medium, credentials = self._build_medium()
113
if 'hpss' in debug.debug_flags:
114
trace.mutter('hpss: Built a new medium: %s',
115
medium.__class__.__name__)
116
self._shared_connection = transport._SharedConnection(medium,
120
# No medium was specified, so share the medium from the
122
medium = self._shared_connection.connection
124
raise AssertionError(
125
"Both _from_transport (%r) and medium (%r) passed to "
126
"RemoteTransport.__init__, but these parameters are mutally "
127
"exclusive." % (_from_transport, medium))
130
self._client = client._SmartClient(medium)
132
self._client = _client
134
def _build_medium(self):
135
"""Create the medium if _from_transport does not provide one.
137
The medium is analogous to the connection for ConnectedTransport: it
138
allows connection sharing.
143
def _report_activity(self, bytes, direction):
144
"""See Transport._report_activity.
146
Does nothing; the smart medium will report activity triggered by a
151
def is_readonly(self):
152
"""Smart server transport can do read/write file operations."""
154
resp = self._call2(b'Transport.is_readonly')
155
except errors.UnknownSmartMethod:
156
# XXX: nasty hack: servers before 0.16 don't have a
157
# 'Transport.is_readonly' verb, so we do what clients before 0.16
160
if resp == (b'yes', ):
162
elif resp == (b'no', ):
165
raise errors.UnexpectedSmartServerResponse(resp)
167
def get_smart_client(self):
168
return self._get_connection()
170
def get_smart_medium(self):
171
return self._get_connection()
173
def _remote_path(self, relpath):
174
"""Returns the Unicode version of the absolute path for relpath."""
175
path = urlutils.URL._combine_paths(self._parsed_url.path, relpath)
176
if not isinstance(path, bytes):
180
def _call(self, method, *args):
181
resp = self._call2(method, *args)
182
self._ensure_ok(resp)
184
def _call2(self, method, *args):
185
"""Call a method on the remote server."""
187
return self._client.call(method, *args)
188
except errors.ErrorFromSmartServer as err:
189
# The first argument, if present, is always a path.
191
context = {'relpath': args[0].decode('utf-8')}
194
self._translate_error(err, **context)
196
def _call_with_body_bytes(self, method, args, body):
197
"""Call a method on the remote server with body bytes."""
199
return self._client.call_with_body_bytes(method, args, body)
200
except errors.ErrorFromSmartServer as err:
201
# The first argument, if present, is always a path.
203
context = {'relpath': args[0]}
206
self._translate_error(err, **context)
208
def has(self, relpath):
209
"""Indicate whether a remote file of the given name exists or not.
211
:see: Transport.has()
213
resp = self._call2(b'has', self._remote_path(relpath))
214
if resp == (b'yes', ):
216
elif resp == (b'no', ):
219
raise errors.UnexpectedSmartServerResponse(resp)
221
def get(self, relpath):
222
"""Return file-like object reading the contents of a remote file.
224
:see: Transport.get_bytes()/get_file()
226
return BytesIO(self.get_bytes(relpath))
228
def get_bytes(self, relpath):
229
remote = self._remote_path(relpath)
231
resp, response_handler = self._client.call_expecting_body(
233
except errors.ErrorFromSmartServer as err:
234
self._translate_error(err, relpath)
235
if resp != (b'ok', ):
236
response_handler.cancel_read_body()
237
raise errors.UnexpectedSmartServerResponse(resp)
238
return response_handler.read_body_bytes()
240
def _serialise_optional_mode(self, mode):
244
return ('%d' % mode).encode('ascii')
246
def mkdir(self, relpath, mode=None):
247
resp = self._call2(b'mkdir', self._remote_path(relpath),
248
self._serialise_optional_mode(mode))
250
def open_write_stream(self, relpath, mode=None):
251
"""See Transport.open_write_stream."""
252
self.put_bytes(relpath, b"", mode)
253
result = transport.AppendBasedFileStream(self, relpath)
254
transport._file_streams[self.abspath(relpath)] = result
257
def put_bytes(self, relpath, raw_bytes, mode=None):
258
if not isinstance(raw_bytes, bytes):
260
'raw_bytes must be bytes string, not %s' % type(raw_bytes))
261
resp = self._call_with_body_bytes(
263
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
265
self._ensure_ok(resp)
266
return len(raw_bytes)
268
def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None,
269
create_parent_dir=False,
271
"""See Transport.put_bytes_non_atomic."""
272
# FIXME: no encoding in the transport!
273
create_parent_str = b'F'
274
if create_parent_dir:
275
create_parent_str = b'T'
277
resp = self._call_with_body_bytes(
279
(self._remote_path(relpath), self._serialise_optional_mode(mode),
280
create_parent_str, self._serialise_optional_mode(dir_mode)),
282
self._ensure_ok(resp)
284
def put_file(self, relpath, upload_file, mode=None):
285
# its not ideal to seek back, but currently put_non_atomic_file depends
286
# on transports not reading before failing - which is a faulty
287
# assumption I think - RBC 20060915
288
pos = upload_file.tell()
290
return self.put_bytes(relpath, upload_file.read(), mode)
292
upload_file.seek(pos)
295
def put_file_non_atomic(self, relpath, f, mode=None,
296
create_parent_dir=False,
298
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
299
create_parent_dir=create_parent_dir,
302
def append_file(self, relpath, from_file, mode=None):
303
return self.append_bytes(relpath, from_file.read(), mode)
305
def append_bytes(self, relpath, bytes, mode=None):
306
resp = self._call_with_body_bytes(
308
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
310
if resp[0] == b'appended':
312
raise errors.UnexpectedSmartServerResponse(resp)
314
def delete(self, relpath):
315
resp = self._call2(b'delete', self._remote_path(relpath))
316
self._ensure_ok(resp)
318
def external_url(self):
319
"""See breezy.transport.Transport.external_url."""
320
# the external path for RemoteTransports is the base
323
def recommended_page_size(self):
324
"""Return the recommended page size for this transport."""
327
def _readv(self, relpath, offsets):
331
offsets = list(offsets)
333
sorted_offsets = sorted(offsets)
334
coalesced = list(self._coalesce_offsets(sorted_offsets,
335
limit=self._max_readv_combine,
336
fudge_factor=self._bytes_to_read_before_seek,
337
max_size=self._max_readv_bytes))
339
# now that we've coallesced things, avoid making enormous requests
344
if c.length + cur_len > self._max_readv_bytes:
345
requests.append(cur_request)
349
cur_request.append(c)
352
requests.append(cur_request)
353
if 'hpss' in debug.debug_flags:
354
trace.mutter('%s.readv %s offsets => %s coalesced'
355
' => %s requests (%s)',
356
self.__class__.__name__, len(offsets), len(coalesced),
357
len(requests), sum(map(len, requests)))
358
# Cache the results, but only until they have been fulfilled
360
# turn the list of offsets into a single stack to iterate
361
offset_stack = iter(offsets)
362
# using a list so it can be modified when passing down and coming back
363
next_offset = [next(offset_stack)]
364
for cur_request in requests:
366
result = self._client.call_with_body_readv_array(
367
(b'readv', self._remote_path(relpath),),
368
[(c.start, c.length) for c in cur_request])
369
resp, response_handler = result
370
except errors.ErrorFromSmartServer as err:
371
self._translate_error(err, relpath)
373
if resp[0] != b'readv':
374
# This should raise an exception
375
response_handler.cancel_read_body()
376
raise errors.UnexpectedSmartServerResponse(resp)
378
for res in self._handle_response(offset_stack, cur_request,
384
def _handle_response(self, offset_stack, coalesced, response_handler,
385
data_map, next_offset):
386
cur_offset_and_size = next_offset[0]
387
# FIXME: this should know how many bytes are needed, for clarity.
388
data = response_handler.read_body_bytes()
390
for c_offset in coalesced:
391
if len(data) < c_offset.length:
392
raise errors.ShortReadvError(relpath, c_offset.start,
393
c_offset.length, actual=len(data))
394
for suboffset, subsize in c_offset.ranges:
395
key = (c_offset.start + suboffset, subsize)
396
this_data = data[data_offset + suboffset:
397
data_offset + suboffset + subsize]
398
# Special case when the data is in-order, rather than packing
399
# into a map and then back out again. Benchmarking shows that
400
# this has 100% hit rate, but leave in the data_map work just
402
# TODO: Could we get away with using buffer() to avoid the
403
# memory copy? Callers would need to realize they may
404
# not have a real string.
405
if key == cur_offset_and_size:
406
yield cur_offset_and_size[0], this_data
408
cur_offset_and_size = next_offset[0] = next(
410
except StopIteration:
413
data_map[key] = this_data
414
data_offset += c_offset.length
416
# Now that we've read some data, see if we can yield anything back
417
while cur_offset_and_size in data_map:
418
this_data = data_map.pop(cur_offset_and_size)
419
yield cur_offset_and_size[0], this_data
421
cur_offset_and_size = next_offset[0] = next(offset_stack)
422
except StopIteration:
425
def rename(self, rel_from, rel_to):
426
self._call(b'rename',
427
self._remote_path(rel_from),
428
self._remote_path(rel_to))
430
def move(self, rel_from, rel_to):
432
self._remote_path(rel_from),
433
self._remote_path(rel_to))
435
def rmdir(self, relpath):
436
resp = self._call(b'rmdir', self._remote_path(relpath))
438
def _ensure_ok(self, resp):
440
raise errors.UnexpectedSmartServerResponse(resp)
442
def _translate_error(self, err, relpath=None):
443
remote._translate_error(err, path=relpath)
445
def disconnect(self):
446
m = self.get_smart_medium()
450
def stat(self, relpath):
451
resp = self._call2(b'stat', self._remote_path(relpath))
452
if resp[0] == b'stat':
453
return _SmartStat(int(resp[1]), int(resp[2], 8))
454
raise errors.UnexpectedSmartServerResponse(resp)
456
# def lock_read(self, relpath):
457
# """Lock the given file for shared (read) access.
458
# :return: A lock object, which should be passed to Transport.unlock()
460
# The old RemoteBranch ignore lock for reading, so we will
461
# continue that tradition and return a bogus lock object.
462
# class BogusLock(object):
463
# def __init__(self, path):
467
# return BogusLock(relpath)
472
def list_dir(self, relpath):
473
resp = self._call2(b'list_dir', self._remote_path(relpath))
474
if resp[0] == b'names':
475
return [name.decode('utf-8') if PY3 else name for name in resp[1:]]
476
raise errors.UnexpectedSmartServerResponse(resp)
478
def iter_files_recursive(self):
479
resp = self._call2(b'iter_files_recursive', self._remote_path(''))
480
if resp[0] == b'names':
481
return [name.decode('utf-8') if PY3 else name for name in resp[1:]]
482
raise errors.UnexpectedSmartServerResponse(resp)
485
class RemoteTCPTransport(RemoteTransport):
486
"""Connection to smart server over plain tcp.
488
This is essentially just a factory to get 'RemoteTransport(url,
489
SmartTCPClientMedium).
492
def _build_medium(self):
493
client_medium = medium.SmartTCPClientMedium(
494
self._parsed_url.host, self._parsed_url.port, self.base)
495
return client_medium, None
498
class RemoteTCPTransportV2Only(RemoteTransport):
499
"""Connection to smart server over plain tcp with the client hard-coded to
500
assume protocol v2 and remote server version <= 1.6.
502
This should only be used for testing.
505
def _build_medium(self):
506
client_medium = medium.SmartTCPClientMedium(
507
self._parsed_url.host, self._parsed_url.port, self.base)
508
client_medium._protocol_version = 2
509
client_medium._remember_remote_is_before((1, 6))
510
return client_medium, None
513
class RemoteSSHTransport(RemoteTransport):
514
"""Connection to smart server over SSH.
516
This is essentially just a factory to get 'RemoteTransport(url,
517
SmartSSHClientMedium).
520
def _build_medium(self):
521
location_config = config.LocationConfig(self.base)
522
bzr_remote_path = location_config.get_bzr_remote_path()
523
user = self._parsed_url.user
525
auth = config.AuthenticationConfig()
526
user = auth.get_user('ssh', self._parsed_url.host,
527
self._parsed_url.port)
528
ssh_params = medium.SSHParams(self._parsed_url.host,
529
self._parsed_url.port, user, self._parsed_url.password,
531
client_medium = medium.SmartSSHClientMedium(self.base, ssh_params)
532
return client_medium, (user, self._parsed_url.password)
535
class RemoteHTTPTransport(RemoteTransport):
536
"""Just a way to connect between a bzr+http:// url and http://.
538
This connection operates slightly differently than the RemoteSSHTransport.
539
It uses a plain http:// transport underneath, which defines what remote
540
.bzr/smart URL we are connected to. From there, all paths that are sent are
541
sent as relative paths, this way, the remote side can properly
542
de-reference them, since it is likely doing rewrite rules to translate an
543
HTTP path into a local path.
546
def __init__(self, base, _from_transport=None, http_transport=None):
547
if http_transport is None:
548
# FIXME: the password may be lost here because it appears in the
549
# url only for an intial construction (when the url came from the
551
http_url = base[len('bzr+'):]
552
self._http_transport = transport.get_transport_from_url(http_url)
554
self._http_transport = http_transport
555
super(RemoteHTTPTransport, self).__init__(
556
base, _from_transport=_from_transport)
558
def _build_medium(self):
559
# We let http_transport take care of the credentials
560
return self._http_transport.get_smart_medium(), None
562
def _remote_path(self, relpath):
563
"""After connecting, HTTP Transport only deals in relative URLs."""
564
# Adjust the relpath based on which URL this smart transport is
566
http_base = urlutils.normalize_url(self.get_smart_medium().base)
567
url = urlutils.join(self.base[len('bzr+'):], relpath)
568
url = urlutils.normalize_url(url)
569
return urlutils.relative_url(http_base, url)
571
def clone(self, relative_url):
572
"""Make a new RemoteHTTPTransport related to me.
574
This is re-implemented rather than using the default
575
RemoteTransport.clone() because we must be careful about the underlying
578
Also, the cloned smart transport will POST to the same .bzr/smart
579
location as this transport (although obviously the relative paths in the
580
smart requests may be different). This is so that the server doesn't
581
have to handle .bzr/smart requests at arbitrary places inside .bzr
582
directories, just at the initial URL the user uses.
585
abs_url = self.abspath(relative_url)
588
return RemoteHTTPTransport(abs_url,
589
_from_transport=self,
590
http_transport=self._http_transport)
592
def _redirected_to(self, source, target):
593
"""See transport._redirected_to"""
594
redirected = self._http_transport._redirected_to(source, target)
595
if (redirected is not None
596
and isinstance(redirected, type(self._http_transport))):
597
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
598
http_transport=redirected)
600
# Either None or a transport for a different protocol
604
class HintingSSHTransport(transport.Transport):
605
"""Simple transport that handles ssh:// and points out bzr+ssh:// and git+ssh://."""
607
# TODO(jelmer): Implement support for detecting whether the repository at the
608
# other end is a git or bzr repository.
610
def __init__(self, url):
611
raise errors.UnsupportedProtocol(
612
url, 'Use bzr+ssh for Bazaar operations over SSH, e.g. "bzr+%s". '
613
'Use git+ssh for Git operations over SSH, e.g. "git+%s".' % (url, url))
616
def get_test_permutations():
617
"""Return (transport, server) permutations for testing."""
618
# We may need a little more test framework support to construct an
619
# appropriate RemoteTransport in the future.
620
from ..tests import test_server
621
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
added
removed
breezy/transport/remote.py
