1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
36
from bzrlib.smart import client, medium
37
from bzrlib.symbol_versioning import (deprecated_method, one_four)
40
class _SmartStat(object):
42
def __init__(self, size, mode):
47
class RemoteTransport(transport.ConnectedTransport):
48
"""Connection to a smart server.
50
The connection holds references to the medium that can be used to send
51
requests to the server.
53
The connection has a notion of the current directory to which it's
54
connected; this is incorporated in filenames passed to the server.
56
This supports some higher-level RPC operations and can also be treated
57
like a Transport to do file-like operations.
59
The connection can be made over a tcp socket, an ssh pipe or a series of
60
http requests. There are concrete subclasses for each type:
61
RemoteTCPTransport, etc.
64
# When making a readv request, cap it at requesting 5MB of data
65
_max_readv_bytes = 5*1024*1024
67
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
68
# responsibilities: Put those on SmartClient or similar. This is vital for
69
# the ability to support multiple versions of the smart protocol over time:
70
# RemoteTransport is an adapter from the Transport object model to the
71
# SmartClient model, not an encoder.
73
# FIXME: the medium parameter should be private, only the tests requires
74
# it. It may be even clearer to define a TestRemoteTransport that handles
75
# the specific cases of providing a _client and/or a _medium, and leave
76
# RemoteTransport as an abstract class.
77
def __init__(self, url, _from_transport=None, medium=None, _client=None):
80
:param _from_transport: Another RemoteTransport instance that this
81
one is being cloned from. Attributes such as the medium will
84
:param medium: The medium to use for this RemoteTransport. If None,
85
the medium from the _from_transport is shared. If both this
86
and _from_transport are None, a new medium will be built.
87
_from_transport and medium cannot both be specified.
89
:param _client: Override the _SmartClient used by this transport. This
90
should only be used for testing purposes; normally this is
91
determined from the medium.
93
super(RemoteTransport, self).__init__(
94
url, _from_transport=_from_transport)
96
# The medium is the connection, except when we need to share it with
97
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
98
# what we want to share is really the shared connection.
100
if (_from_transport is not None
101
and isinstance(_from_transport, RemoteTransport)):
102
_client = _from_transport._client
103
elif _from_transport is None:
104
# If no _from_transport is specified, we need to intialize the
108
medium, credentials = self._build_medium()
109
if 'hpss' in debug.debug_flags:
110
trace.mutter('hpss: Built a new medium: %s',
111
medium.__class__.__name__)
112
self._shared_connection = transport._SharedConnection(medium,
116
# No medium was specified, so share the medium from the
118
medium = self._shared_connection.connection
120
raise AssertionError(
121
"Both _from_transport (%r) and medium (%r) passed to "
122
"RemoteTransport.__init__, but these parameters are mutally "
123
"exclusive." % (_from_transport, medium))
126
self._client = client._SmartClient(medium)
128
self._client = _client
130
def _build_medium(self):
131
"""Create the medium if _from_transport does not provide one.
133
The medium is analogous to the connection for ConnectedTransport: it
134
allows connection sharing.
139
def _report_activity(self, bytes, direction):
140
"""See Transport._report_activity.
142
Does nothing; the smart medium will report activity triggered by a
147
def is_readonly(self):
148
"""Smart server transport can do read/write file operations."""
150
resp = self._call2('Transport.is_readonly')
151
except errors.UnknownSmartMethod:
152
# XXX: nasty hack: servers before 0.16 don't have a
153
# 'Transport.is_readonly' verb, so we do what clients before 0.16
156
if resp == ('yes', ):
158
elif resp == ('no', ):
161
raise errors.UnexpectedSmartServerResponse(resp)
163
def get_smart_client(self):
164
return self._get_connection()
166
def get_smart_medium(self):
167
return self._get_connection()
169
@deprecated_method(one_four)
170
def get_shared_medium(self):
171
return self._get_shared_connection()
173
def _remote_path(self, relpath):
174
"""Returns the Unicode version of the absolute path for relpath."""
175
return self._combine_paths(self._path, relpath)
177
def _call(self, method, *args):
178
resp = self._call2(method, *args)
179
self._ensure_ok(resp)
181
def _call2(self, method, *args):
182
"""Call a method on the remote server."""
184
return self._client.call(method, *args)
185
except errors.ErrorFromSmartServer, err:
186
# The first argument, if present, is always a path.
188
context = {'relpath': args[0]}
191
self._translate_error(err, **context)
193
def _call_with_body_bytes(self, method, args, body):
194
"""Call a method on the remote server with body bytes."""
196
return self._client.call_with_body_bytes(method, args, body)
197
except errors.ErrorFromSmartServer, err:
198
# The first argument, if present, is always a path.
200
context = {'relpath': args[0]}
203
self._translate_error(err, **context)
205
def has(self, relpath):
206
"""Indicate whether a remote file of the given name exists or not.
208
:see: Transport.has()
210
resp = self._call2('has', self._remote_path(relpath))
211
if resp == ('yes', ):
213
elif resp == ('no', ):
216
raise errors.UnexpectedSmartServerResponse(resp)
218
def get(self, relpath):
219
"""Return file-like object reading the contents of a remote file.
221
:see: Transport.get_bytes()/get_file()
223
return StringIO(self.get_bytes(relpath))
225
def get_bytes(self, relpath):
226
remote = self._remote_path(relpath)
228
resp, response_handler = self._client.call_expecting_body('get', remote)
229
except errors.ErrorFromSmartServer, err:
230
self._translate_error(err, relpath)
232
response_handler.cancel_read_body()
233
raise errors.UnexpectedSmartServerResponse(resp)
234
return response_handler.read_body_bytes()
236
def _serialise_optional_mode(self, mode):
242
def mkdir(self, relpath, mode=None):
243
resp = self._call2('mkdir', self._remote_path(relpath),
244
self._serialise_optional_mode(mode))
246
def open_write_stream(self, relpath, mode=None):
247
"""See Transport.open_write_stream."""
248
self.put_bytes(relpath, "", mode)
249
result = transport.AppendBasedFileStream(self, relpath)
250
transport._file_streams[self.abspath(relpath)] = result
253
def put_bytes(self, relpath, upload_contents, mode=None):
254
# FIXME: upload_file is probably not safe for non-ascii characters -
255
# should probably just pass all parameters as length-delimited
257
if type(upload_contents) is unicode:
258
# Although not strictly correct, we raise UnicodeEncodeError to be
259
# compatible with other transports.
260
raise UnicodeEncodeError(
261
'undefined', upload_contents, 0, 1,
262
'put_bytes must be given bytes, not unicode.')
263
resp = self._call_with_body_bytes('put',
264
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
266
self._ensure_ok(resp)
267
return len(upload_contents)
269
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
270
create_parent_dir=False,
272
"""See Transport.put_bytes_non_atomic."""
273
# FIXME: no encoding in the transport!
274
create_parent_str = 'F'
275
if create_parent_dir:
276
create_parent_str = 'T'
278
resp = self._call_with_body_bytes(
280
(self._remote_path(relpath), self._serialise_optional_mode(mode),
281
create_parent_str, self._serialise_optional_mode(dir_mode)),
283
self._ensure_ok(resp)
285
def put_file(self, relpath, upload_file, mode=None):
286
# its not ideal to seek back, but currently put_non_atomic_file depends
287
# on transports not reading before failing - which is a faulty
288
# assumption I think - RBC 20060915
289
pos = upload_file.tell()
291
return self.put_bytes(relpath, upload_file.read(), mode)
293
upload_file.seek(pos)
296
def put_file_non_atomic(self, relpath, f, mode=None,
297
create_parent_dir=False,
299
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
300
create_parent_dir=create_parent_dir,
303
def append_file(self, relpath, from_file, mode=None):
304
return self.append_bytes(relpath, from_file.read(), mode)
306
def append_bytes(self, relpath, bytes, mode=None):
307
resp = self._call_with_body_bytes(
309
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
311
if resp[0] == 'appended':
313
raise errors.UnexpectedSmartServerResponse(resp)
315
def delete(self, relpath):
316
resp = self._call2('delete', self._remote_path(relpath))
317
self._ensure_ok(resp)
319
def external_url(self):
320
"""See bzrlib.transport.Transport.external_url."""
321
# the external path for RemoteTransports is the base
324
def recommended_page_size(self):
325
"""Return the recommended page size for this transport."""
328
def _readv(self, relpath, offsets):
332
offsets = list(offsets)
334
sorted_offsets = sorted(offsets)
335
coalesced = list(self._coalesce_offsets(sorted_offsets,
336
limit=self._max_readv_combine,
337
fudge_factor=self._bytes_to_read_before_seek,
338
max_size=self._max_readv_bytes))
340
# now that we've coallesced things, avoid making enormous requests
345
if c.length + cur_len > self._max_readv_bytes:
346
requests.append(cur_request)
350
cur_request.append(c)
353
requests.append(cur_request)
354
if 'hpss' in debug.debug_flags:
355
trace.mutter('%s.readv %s offsets => %s coalesced'
356
' => %s requests (%s)',
357
self.__class__.__name__, len(offsets), len(coalesced),
358
len(requests), sum(map(len, requests)))
359
# Cache the results, but only until they have been fulfilled
361
# turn the list of offsets into a single stack to iterate
362
offset_stack = iter(offsets)
363
# using a list so it can be modified when passing down and coming back
364
next_offset = [offset_stack.next()]
365
for cur_request in requests:
367
result = self._client.call_with_body_readv_array(
368
('readv', self._remote_path(relpath),),
369
[(c.start, c.length) for c in cur_request])
370
resp, response_handler = result
371
except errors.ErrorFromSmartServer, err:
372
self._translate_error(err, relpath)
374
if resp[0] != 'readv':
375
# This should raise an exception
376
response_handler.cancel_read_body()
377
raise errors.UnexpectedSmartServerResponse(resp)
379
for res in self._handle_response(offset_stack, cur_request,
385
def _handle_response(self, offset_stack, coalesced, response_handler,
386
data_map, next_offset):
387
cur_offset_and_size = next_offset[0]
388
# FIXME: this should know how many bytes are needed, for clarity.
389
data = response_handler.read_body_bytes()
391
for c_offset in coalesced:
392
if len(data) < c_offset.length:
393
raise errors.ShortReadvError(relpath, c_offset.start,
394
c_offset.length, actual=len(data))
395
for suboffset, subsize in c_offset.ranges:
396
key = (c_offset.start+suboffset, subsize)
397
this_data = data[data_offset+suboffset:
398
data_offset+suboffset+subsize]
399
# Special case when the data is in-order, rather than packing
400
# into a map and then back out again. Benchmarking shows that
401
# this has 100% hit rate, but leave in the data_map work just
403
# TODO: Could we get away with using buffer() to avoid the
404
# memory copy? Callers would need to realize they may
405
# not have a real string.
406
if key == cur_offset_and_size:
407
yield cur_offset_and_size[0], this_data
408
cur_offset_and_size = next_offset[0] = offset_stack.next()
410
data_map[key] = this_data
411
data_offset += c_offset.length
413
# Now that we've read some data, see if we can yield anything back
414
while cur_offset_and_size in data_map:
415
this_data = data_map.pop(cur_offset_and_size)
416
yield cur_offset_and_size[0], this_data
417
cur_offset_and_size = next_offset[0] = offset_stack.next()
419
def rename(self, rel_from, rel_to):
421
self._remote_path(rel_from),
422
self._remote_path(rel_to))
424
def move(self, rel_from, rel_to):
426
self._remote_path(rel_from),
427
self._remote_path(rel_to))
429
def rmdir(self, relpath):
430
resp = self._call('rmdir', self._remote_path(relpath))
432
def _ensure_ok(self, resp):
434
raise errors.UnexpectedSmartServerResponse(resp)
436
def _translate_error(self, err, relpath=None):
437
remote._translate_error(err, path=relpath)
439
def disconnect(self):
440
self.get_smart_medium().disconnect()
442
def stat(self, relpath):
443
resp = self._call2('stat', self._remote_path(relpath))
444
if resp[0] == 'stat':
445
return _SmartStat(int(resp[1]), int(resp[2], 8))
446
raise errors.UnexpectedSmartServerResponse(resp)
448
## def lock_read(self, relpath):
449
## """Lock the given file for shared (read) access.
450
## :return: A lock object, which should be passed to Transport.unlock()
452
## # The old RemoteBranch ignore lock for reading, so we will
453
## # continue that tradition and return a bogus lock object.
454
## class BogusLock(object):
455
## def __init__(self, path):
459
## return BogusLock(relpath)
464
def list_dir(self, relpath):
465
resp = self._call2('list_dir', self._remote_path(relpath))
466
if resp[0] == 'names':
467
return [name.encode('ascii') for name in resp[1:]]
468
raise errors.UnexpectedSmartServerResponse(resp)
470
def iter_files_recursive(self):
471
resp = self._call2('iter_files_recursive', self._remote_path(''))
472
if resp[0] == 'names':
474
raise errors.UnexpectedSmartServerResponse(resp)
477
class RemoteTCPTransport(RemoteTransport):
478
"""Connection to smart server over plain tcp.
480
This is essentially just a factory to get 'RemoteTransport(url,
481
SmartTCPClientMedium).
484
def _build_medium(self):
485
client_medium = medium.SmartTCPClientMedium(
486
self._host, self._port, self.base)
487
return client_medium, None
490
class RemoteTCPTransportV2Only(RemoteTransport):
491
"""Connection to smart server over plain tcp with the client hard-coded to
492
assume protocol v2 and remote server version <= 1.6.
494
This should only be used for testing.
497
def _build_medium(self):
498
client_medium = medium.SmartTCPClientMedium(
499
self._host, self._port, self.base)
500
client_medium._protocol_version = 2
501
client_medium._remember_remote_is_before((1, 6))
502
return client_medium, None
505
class RemoteSSHTransport(RemoteTransport):
506
"""Connection to smart server over SSH.
508
This is essentially just a factory to get 'RemoteTransport(url,
509
SmartSSHClientMedium).
512
def _build_medium(self):
513
location_config = config.LocationConfig(self.base)
514
bzr_remote_path = location_config.get_bzr_remote_path()
517
auth = config.AuthenticationConfig()
518
user = auth.get_user('ssh', self._host, self._port)
519
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
520
user, self._password, self.base,
521
bzr_remote_path=bzr_remote_path)
522
return client_medium, (user, self._password)
525
class RemoteHTTPTransport(RemoteTransport):
526
"""Just a way to connect between a bzr+http:// url and http://.
528
This connection operates slightly differently than the RemoteSSHTransport.
529
It uses a plain http:// transport underneath, which defines what remote
530
.bzr/smart URL we are connected to. From there, all paths that are sent are
531
sent as relative paths, this way, the remote side can properly
532
de-reference them, since it is likely doing rewrite rules to translate an
533
HTTP path into a local path.
536
def __init__(self, base, _from_transport=None, http_transport=None):
537
if http_transport is None:
538
# FIXME: the password may be lost here because it appears in the
539
# url only for an intial construction (when the url came from the
541
http_url = base[len('bzr+'):]
542
self._http_transport = transport.get_transport(http_url)
544
self._http_transport = http_transport
545
super(RemoteHTTPTransport, self).__init__(
546
base, _from_transport=_from_transport)
548
def _build_medium(self):
549
# We let http_transport take care of the credentials
550
return self._http_transport.get_smart_medium(), None
552
def _remote_path(self, relpath):
553
"""After connecting, HTTP Transport only deals in relative URLs."""
554
# Adjust the relpath based on which URL this smart transport is
556
http_base = urlutils.normalize_url(self.get_smart_medium().base)
557
url = urlutils.join(self.base[len('bzr+'):], relpath)
558
url = urlutils.normalize_url(url)
559
return urlutils.relative_url(http_base, url)
561
def clone(self, relative_url):
562
"""Make a new RemoteHTTPTransport related to me.
564
This is re-implemented rather than using the default
565
RemoteTransport.clone() because we must be careful about the underlying
568
Also, the cloned smart transport will POST to the same .bzr/smart
569
location as this transport (although obviously the relative paths in the
570
smart requests may be different). This is so that the server doesn't
571
have to handle .bzr/smart requests at arbitrary places inside .bzr
572
directories, just at the initial URL the user uses.
575
abs_url = self.abspath(relative_url)
578
return RemoteHTTPTransport(abs_url,
579
_from_transport=self,
580
http_transport=self._http_transport)
582
def _redirected_to(self, source, target):
583
"""See transport._redirected_to"""
584
redirected = self._http_transport._redirected_to(source, target)
585
if (redirected is not None
586
and isinstance(redirected, type(self._http_transport))):
587
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
588
http_transport=redirected)
590
# Either None or a transport for a different protocol
594
class HintingSSHTransport(transport.Transport):
595
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
597
def __init__(self, url):
598
raise errors.UnsupportedProtocol(url,
599
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
602
def get_test_permutations():
603
"""Return (transport, server) permutations for testing."""
604
### We may need a little more test framework support to construct an
605
### appropriate RemoteTransport in the future.
606
from bzrlib.smart import server
607
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
