1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""RemoteTransport client for the smart-server.
19
This module shouldn't be accessed directly. The classes defined here should be
20
imported from bzrlib.smart.
23
__all__ = ['RemoteTransport', 'RemoteTCPTransport', 'RemoteSSHTransport']
25
from cStringIO import StringIO
35
from bzrlib.smart import client, medium
36
from bzrlib.symbol_versioning import (deprecated_method, one_four)
39
class _SmartStat(object):
41
def __init__(self, size, mode):
46
class RemoteTransport(transport.ConnectedTransport):
47
"""Connection to a smart server.
49
The connection holds references to the medium that can be used to send
50
requests to the server.
52
The connection has a notion of the current directory to which it's
53
connected; this is incorporated in filenames passed to the server.
55
This supports some higher-level RPC operations and can also be treated
56
like a Transport to do file-like operations.
58
The connection can be made over a tcp socket, an ssh pipe or a series of
59
http requests. There are concrete subclasses for each type:
60
RemoteTCPTransport, etc.
63
# When making a readv request, cap it at requesting 5MB of data
64
_max_readv_bytes = 5*1024*1024
66
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
67
# responsibilities: Put those on SmartClient or similar. This is vital for
68
# the ability to support multiple versions of the smart protocol over time:
69
# RemoteTransport is an adapter from the Transport object model to the
70
# SmartClient model, not an encoder.
72
# FIXME: the medium parameter should be private, only the tests requires
73
# it. It may be even clearer to define a TestRemoteTransport that handles
74
# the specific cases of providing a _client and/or a _medium, and leave
75
# RemoteTransport as an abstract class.
76
def __init__(self, url, _from_transport=None, medium=None, _client=None):
79
:param _from_transport: Another RemoteTransport instance that this
80
one is being cloned from. Attributes such as the medium will
83
:param medium: The medium to use for this RemoteTransport. If None,
84
the medium from the _from_transport is shared. If both this
85
and _from_transport are None, a new medium will be built.
86
_from_transport and medium cannot both be specified.
88
:param _client: Override the _SmartClient used by this transport. This
89
should only be used for testing purposes; normally this is
90
determined from the medium.
92
super(RemoteTransport, self).__init__(url,
93
_from_transport=_from_transport)
95
# The medium is the connection, except when we need to share it with
96
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
97
# what we want to share is really the shared connection.
99
if _from_transport is None:
100
# If no _from_transport is specified, we need to intialize the
104
medium, credentials = self._build_medium()
105
if 'hpss' in debug.debug_flags:
106
trace.mutter('hpss: Built a new medium: %s',
107
medium.__class__.__name__)
108
self._shared_connection = transport._SharedConnection(medium,
112
# No medium was specified, so share the medium from the
114
medium = self._shared_connection.connection
116
raise AssertionError(
117
"Both _from_transport (%r) and medium (%r) passed to "
118
"RemoteTransport.__init__, but these parameters are mutally "
119
"exclusive." % (_from_transport, medium))
122
self._client = client._SmartClient(medium)
124
self._client = _client
126
def _build_medium(self):
127
"""Create the medium if _from_transport does not provide one.
129
The medium is analogous to the connection for ConnectedTransport: it
130
allows connection sharing.
135
def is_readonly(self):
136
"""Smart server transport can do read/write file operations."""
138
resp = self._call2('Transport.is_readonly')
139
except errors.UnknownSmartMethod:
140
# XXX: nasty hack: servers before 0.16 don't have a
141
# 'Transport.is_readonly' verb, so we do what clients before 0.16
144
if resp == ('yes', ):
146
elif resp == ('no', ):
149
self._translate_error(resp)
150
raise errors.UnexpectedSmartServerResponse(resp)
152
def get_smart_client(self):
153
return self._get_connection()
155
def get_smart_medium(self):
156
return self._get_connection()
158
@deprecated_method(one_four)
159
def get_shared_medium(self):
160
return self._get_shared_connection()
162
def _remote_path(self, relpath):
163
"""Returns the Unicode version of the absolute path for relpath."""
164
return self._combine_paths(self._path, relpath)
166
def _call(self, method, *args):
168
resp = self._call2(method, *args)
169
except errors.ErrorFromSmartServer, err:
170
self._translate_error(err.error_tuple)
171
self._translate_error(resp)
173
def _call2(self, method, *args):
174
"""Call a method on the remote server."""
176
return self._client.call(method, *args)
177
except errors.ErrorFromSmartServer, err:
178
self._translate_error(err.error_tuple)
180
def _call_with_body_bytes(self, method, args, body):
181
"""Call a method on the remote server with body bytes."""
183
return self._client.call_with_body_bytes(method, args, body)
184
except errors.ErrorFromSmartServer, err:
185
self._translate_error(err.error_tuple)
187
def has(self, relpath):
188
"""Indicate whether a remote file of the given name exists or not.
190
:see: Transport.has()
192
resp = self._call2('has', self._remote_path(relpath))
193
if resp == ('yes', ):
195
elif resp == ('no', ):
198
self._translate_error(resp)
200
def get(self, relpath):
201
"""Return file-like object reading the contents of a remote file.
203
:see: Transport.get_bytes()/get_file()
205
return StringIO(self.get_bytes(relpath))
207
def get_bytes(self, relpath):
208
remote = self._remote_path(relpath)
210
resp, response_handler = self._client.call_expecting_body('get', remote)
211
except errors.ErrorFromSmartServer, err:
212
self._translate_error(err.error_tuple, relpath)
214
response_handler.cancel_read_body()
215
raise errors.UnexpectedSmartServerResponse(resp)
216
return response_handler.read_body_bytes()
218
def _serialise_optional_mode(self, mode):
224
def mkdir(self, relpath, mode=None):
225
resp = self._call2('mkdir', self._remote_path(relpath),
226
self._serialise_optional_mode(mode))
227
self._translate_error(resp)
229
def open_write_stream(self, relpath, mode=None):
230
"""See Transport.open_write_stream."""
231
self.put_bytes(relpath, "", mode)
232
result = transport.AppendBasedFileStream(self, relpath)
233
transport._file_streams[self.abspath(relpath)] = result
236
def put_bytes(self, relpath, upload_contents, mode=None):
237
# FIXME: upload_file is probably not safe for non-ascii characters -
238
# should probably just pass all parameters as length-delimited
240
if type(upload_contents) is unicode:
241
# Although not strictly correct, we raise UnicodeEncodeError to be
242
# compatible with other transports.
243
raise UnicodeEncodeError(
244
'undefined', upload_contents, 0, 1,
245
'put_bytes must be given bytes, not unicode.')
246
resp = self._call_with_body_bytes('put',
247
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
249
self._translate_error(resp)
250
return len(upload_contents)
252
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
253
create_parent_dir=False,
255
"""See Transport.put_bytes_non_atomic."""
256
# FIXME: no encoding in the transport!
257
create_parent_str = 'F'
258
if create_parent_dir:
259
create_parent_str = 'T'
261
resp = self._call_with_body_bytes(
263
(self._remote_path(relpath), self._serialise_optional_mode(mode),
264
create_parent_str, self._serialise_optional_mode(dir_mode)),
266
self._translate_error(resp)
268
def put_file(self, relpath, upload_file, mode=None):
269
# its not ideal to seek back, but currently put_non_atomic_file depends
270
# on transports not reading before failing - which is a faulty
271
# assumption I think - RBC 20060915
272
pos = upload_file.tell()
274
return self.put_bytes(relpath, upload_file.read(), mode)
276
upload_file.seek(pos)
279
def put_file_non_atomic(self, relpath, f, mode=None,
280
create_parent_dir=False,
282
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
283
create_parent_dir=create_parent_dir,
286
def append_file(self, relpath, from_file, mode=None):
287
return self.append_bytes(relpath, from_file.read(), mode)
289
def append_bytes(self, relpath, bytes, mode=None):
290
resp = self._call_with_body_bytes(
292
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
294
if resp[0] == 'appended':
296
self._translate_error(resp)
298
def delete(self, relpath):
299
resp = self._call2('delete', self._remote_path(relpath))
300
self._translate_error(resp)
302
def external_url(self):
303
"""See bzrlib.transport.Transport.external_url."""
304
# the external path for RemoteTransports is the base
307
def recommended_page_size(self):
308
"""Return the recommended page size for this transport."""
311
def _readv(self, relpath, offsets):
315
offsets = list(offsets)
317
sorted_offsets = sorted(offsets)
318
coalesced = list(self._coalesce_offsets(sorted_offsets,
319
limit=self._max_readv_combine,
320
fudge_factor=self._bytes_to_read_before_seek))
322
# now that we've coallesced things, avoid making enormous requests
327
if c.length + cur_len > self._max_readv_bytes:
328
requests.append(cur_request)
332
cur_request.append(c)
335
requests.append(cur_request)
336
if 'hpss' in debug.debug_flags:
337
trace.mutter('%s.readv %s offsets => %s coalesced'
338
' => %s requests (%s)',
339
self.__class__.__name__, len(offsets), len(coalesced),
340
len(requests), sum(map(len, requests)))
341
# Cache the results, but only until they have been fulfilled
343
# turn the list of offsets into a single stack to iterate
344
offset_stack = iter(offsets)
345
# using a list so it can be modified when passing down and coming back
346
next_offset = [cur_offset_and_size.next()]
347
for cur_request in requests:
349
result = self._client.call_with_body_readv_array(
350
('readv', self._remote_path(relpath),),
351
[(c.start, c.length) for c in cur_request])
352
resp, response_handler = result
353
except errors.ErrorFromSmartServer, err:
354
self._translate_error(err.error_tuple)
356
if resp[0] != 'readv':
357
# This should raise an exception
358
response_handler.cancel_read_body()
359
raise errors.UnexpectedSmartServerResponse(resp)
361
for res in self._handle_response(offset_stack, cur_request,
367
def _handle_response(self, offset_stack, coalesced, response_handler,
368
data_map, next_offset):
369
cur_offset_and_size = next_offset[0]
370
# FIXME: this should know how many bytes are needed, for clarity.
371
data = response_handler.read_body_bytes()
373
for c_offset in coalesced:
374
if len(data) < c_offset.length:
375
raise errors.ShortReadvError(relpath, c_offset.start,
376
c_offset.length, actual=len(data))
377
for suboffset, subsize in c_offset.ranges:
378
key = (c_offset.start+suboffset, subsize)
379
this_data = data[data_offset+suboffset:
380
data_offset+suboffset+subsize]
381
# Special case when the data is in-order, rather than packing
382
# into a map and then back out again. Benchmarking shows that
383
# this has 100% hit rate, but leave in the data_map work just
385
# TODO: Could we get away with using buffer() to avoid the
386
# memory copy? Callers would need to realize they may
387
# not have a real string.
388
if key == cur_offset_and_size:
389
yield cur_offset_and_size[0], this_data
390
cur_offset_and_size = next_offset[0] = offset_stack.next()
392
data_map[key] = this_data
393
data_offset += c_offset.length
395
# Now that we've read some data, see if we can yield anything back
396
while cur_offset_and_size in data_map:
397
this_data = data_map.pop(cur_offset_and_size)
398
yield cur_offset_and_size[0], this_data
399
cur_offset_and_size = next_offset[0] = offset_stack.next()
401
def rename(self, rel_from, rel_to):
403
self._remote_path(rel_from),
404
self._remote_path(rel_to))
406
def move(self, rel_from, rel_to):
408
self._remote_path(rel_from),
409
self._remote_path(rel_to))
411
def rmdir(self, relpath):
412
resp = self._call('rmdir', self._remote_path(relpath))
414
def _translate_error(self, resp, orig_path=None):
415
"""Raise an exception from a response"""
422
elif what == 'NoSuchFile':
423
if orig_path is not None:
424
error_path = orig_path
427
raise errors.NoSuchFile(error_path)
428
elif what == 'error':
429
raise errors.SmartProtocolError(unicode(resp[1]))
430
elif what == 'FileExists':
431
raise errors.FileExists(resp[1])
432
elif what == 'DirectoryNotEmpty':
433
raise errors.DirectoryNotEmpty(resp[1])
434
elif what == 'ShortReadvError':
435
raise errors.ShortReadvError(resp[1], int(resp[2]),
436
int(resp[3]), int(resp[4]))
437
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
438
encoding = str(resp[1]) # encoding must always be a string
442
reason = str(resp[5]) # reason must always be a string
443
if val.startswith('u:'):
444
val = val[2:].decode('utf-8')
445
elif val.startswith('s:'):
446
val = val[2:].decode('base64')
447
if what == 'UnicodeDecodeError':
448
raise UnicodeDecodeError(encoding, val, start, end, reason)
449
elif what == 'UnicodeEncodeError':
450
raise UnicodeEncodeError(encoding, val, start, end, reason)
451
elif what == "ReadOnlyError":
452
raise errors.TransportNotPossible('readonly transport')
453
elif what == "ReadError":
454
if orig_path is not None:
455
error_path = orig_path
458
raise errors.ReadError(error_path)
459
elif what == "PermissionDenied":
460
if orig_path is not None:
461
error_path = orig_path
464
raise errors.PermissionDenied(error_path)
466
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
468
def disconnect(self):
469
self.get_smart_medium().disconnect()
471
def stat(self, relpath):
472
resp = self._call2('stat', self._remote_path(relpath))
473
if resp[0] == 'stat':
474
return _SmartStat(int(resp[1]), int(resp[2], 8))
476
self._translate_error(resp)
478
## def lock_read(self, relpath):
479
## """Lock the given file for shared (read) access.
480
## :return: A lock object, which should be passed to Transport.unlock()
482
## # The old RemoteBranch ignore lock for reading, so we will
483
## # continue that tradition and return a bogus lock object.
484
## class BogusLock(object):
485
## def __init__(self, path):
489
## return BogusLock(relpath)
494
def list_dir(self, relpath):
495
resp = self._call2('list_dir', self._remote_path(relpath))
496
if resp[0] == 'names':
497
return [name.encode('ascii') for name in resp[1:]]
499
self._translate_error(resp)
501
def iter_files_recursive(self):
502
resp = self._call2('iter_files_recursive', self._remote_path(''))
503
if resp[0] == 'names':
506
self._translate_error(resp)
509
class RemoteTCPTransport(RemoteTransport):
510
"""Connection to smart server over plain tcp.
512
This is essentially just a factory to get 'RemoteTransport(url,
513
SmartTCPClientMedium).
516
def _build_medium(self):
517
client_medium = medium.SmartTCPClientMedium(
518
self._host, self._port, self.base)
519
return client_medium, None
522
class RemoteTCPTransportV2Only(RemoteTransport):
523
"""Connection to smart server over plain tcp with the client hard-coded to
524
assume protocol v2 and remote server version <= 1.6.
526
This should only be used for testing.
529
def _build_medium(self):
530
client_medium = medium.SmartTCPClientMedium(
531
self._host, self._port, self.base)
532
client_medium._protocol_version = 2
533
client_medium._remember_remote_is_before((1, 6))
534
return client_medium, None
537
class RemoteSSHTransport(RemoteTransport):
538
"""Connection to smart server over SSH.
540
This is essentially just a factory to get 'RemoteTransport(url,
541
SmartSSHClientMedium).
544
def _build_medium(self):
545
location_config = config.LocationConfig(self.base)
546
bzr_remote_path = location_config.get_bzr_remote_path()
549
auth = config.AuthenticationConfig()
550
user = auth.get_user('ssh', self._host, self._port)
551
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
552
user, self._password, self.base,
553
bzr_remote_path=bzr_remote_path)
554
return client_medium, (user, self._password)
557
class RemoteHTTPTransport(RemoteTransport):
558
"""Just a way to connect between a bzr+http:// url and http://.
560
This connection operates slightly differently than the RemoteSSHTransport.
561
It uses a plain http:// transport underneath, which defines what remote
562
.bzr/smart URL we are connected to. From there, all paths that are sent are
563
sent as relative paths, this way, the remote side can properly
564
de-reference them, since it is likely doing rewrite rules to translate an
565
HTTP path into a local path.
568
def __init__(self, base, _from_transport=None, http_transport=None):
569
if http_transport is None:
570
# FIXME: the password may be lost here because it appears in the
571
# url only for an intial construction (when the url came from the
573
http_url = base[len('bzr+'):]
574
self._http_transport = transport.get_transport(http_url)
576
self._http_transport = http_transport
577
super(RemoteHTTPTransport, self).__init__(
578
base, _from_transport=_from_transport)
580
def _build_medium(self):
581
# We let http_transport take care of the credentials
582
return self._http_transport.get_smart_medium(), None
584
def _remote_path(self, relpath):
585
"""After connecting, HTTP Transport only deals in relative URLs."""
586
# Adjust the relpath based on which URL this smart transport is
588
http_base = urlutils.normalize_url(self.get_smart_medium().base)
589
url = urlutils.join(self.base[len('bzr+'):], relpath)
590
url = urlutils.normalize_url(url)
591
return urlutils.relative_url(http_base, url)
593
def clone(self, relative_url):
594
"""Make a new RemoteHTTPTransport related to me.
596
This is re-implemented rather than using the default
597
RemoteTransport.clone() because we must be careful about the underlying
600
Also, the cloned smart transport will POST to the same .bzr/smart
601
location as this transport (although obviously the relative paths in the
602
smart requests may be different). This is so that the server doesn't
603
have to handle .bzr/smart requests at arbitrary places inside .bzr
604
directories, just at the initial URL the user uses.
607
abs_url = self.abspath(relative_url)
610
return RemoteHTTPTransport(abs_url,
611
_from_transport=self,
612
http_transport=self._http_transport)
615
def get_test_permutations():
616
"""Return (transport, server) permutations for testing."""
617
### We may need a little more test framework support to construct an
618
### appropriate RemoteTransport in the future.
619
from bzrlib.smart import server
620
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
