1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
from cStringIO import StringIO
25
from bzrlib.smart.protocol import SmartClientRequestProtocolOne
26
from bzrlib.smart.medium import SmartTCPClientMedium, SmartSSHClientMedium
28
# must do this otherwise urllib can't parse the urls properly :(
29
for scheme in ['ssh', 'bzr', 'bzr+loopback', 'bzr+ssh', 'bzr+http']:
30
transport.register_urlparse_netloc_protocol(scheme)
34
# Port 4155 is the default port for bzr://, registered with IANA.
35
BZR_DEFAULT_PORT = 4155
38
class SmartStat(object):
40
def __init__(self, size, mode):
45
class RemoteTransport(transport.Transport):
46
"""Connection to a smart server.
48
The connection holds references to pipes that can be used to send requests
51
The connection has a notion of the current directory to which it's
52
connected; this is incorporated in filenames passed to the server.
54
This supports some higher-level RPC operations and can also be treated
55
like a Transport to do file-like operations.
57
The connection can be made over a tcp socket, or (in future) an ssh pipe
58
or a series of http requests. There are concrete subclasses for each
59
type: RemoteTCPTransport, etc.
62
# IMPORTANT FOR IMPLEMENTORS: RemoteTransport MUST NOT be given encoding
63
# responsibilities: Put those on SmartClient or similar. This is vital for
64
# the ability to support multiple versions of the smart protocol over time:
65
# RemoteTransport is an adapter from the Transport object model to the
66
# SmartClient model, not an encoder.
68
def __init__(self, url, clone_from=None, medium=None):
71
:param medium: The medium to use for this RemoteTransport. This must be
72
supplied if clone_from is None.
74
### Technically super() here is faulty because Transport's __init__
75
### fails to take 2 parameters, and if super were to choose a silly
76
### initialisation order things would blow up.
77
if not url.endswith('/'):
79
super(RemoteTransport, self).__init__(url)
80
self._scheme, self._username, self._password, self._host, self._port, self._path = \
81
transport.split_url(url)
82
if clone_from is None:
85
# credentials may be stripped from the base in some circumstances
86
# as yet to be clearly defined or documented, so copy them.
87
self._username = clone_from._username
88
# reuse same connection
89
self._medium = clone_from._medium
90
assert self._medium is not None
92
def abspath(self, relpath):
93
"""Return the full url to the given relative path.
95
@param relpath: the relative path or path components
96
@type relpath: str or list
98
return self._unparse_url(self._remote_path(relpath))
100
def clone(self, relative_url):
101
"""Make a new RemoteTransport related to me, sharing the same connection.
103
This essentially opens a handle on a different remote directory.
105
if relative_url is None:
106
return RemoteTransport(self.base, self)
108
return RemoteTransport(self.abspath(relative_url), self)
110
def is_readonly(self):
111
"""Smart server transport can do read/write file operations."""
114
def get_smart_client(self):
117
def get_smart_medium(self):
120
def _unparse_url(self, path):
121
"""Return URL for a path.
123
:see: SFTPUrlHandling._unparse_url
125
# TODO: Eventually it should be possible to unify this with
126
# SFTPUrlHandling._unparse_url?
129
path = urllib.quote(path)
130
netloc = urllib.quote(self._host)
131
if self._username is not None:
132
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
133
if self._port is not None:
134
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((self._scheme, netloc, path, '', '', ''))
137
def _remote_path(self, relpath):
138
"""Returns the Unicode version of the absolute path for relpath."""
139
return self._combine_paths(self._path, relpath)
141
def _call(self, method, *args):
142
resp = self._call2(method, *args)
143
self._translate_error(resp)
145
def _call2(self, method, *args):
146
"""Call a method on the remote server."""
147
protocol = SmartClientRequestProtocolOne(self._medium.get_request())
148
protocol.call(method, *args)
149
return protocol.read_response_tuple()
151
def _call_with_body_bytes(self, method, args, body):
152
"""Call a method on the remote server with body bytes."""
153
protocol = SmartClientRequestProtocolOne(self._medium.get_request())
154
protocol.call_with_body_bytes((method, ) + args, body)
155
return protocol.read_response_tuple()
157
def has(self, relpath):
158
"""Indicate whether a remote file of the given name exists or not.
160
:see: Transport.has()
162
resp = self._call2('has', self._remote_path(relpath))
163
if resp == ('yes', ):
165
elif resp == ('no', ):
168
self._translate_error(resp)
170
def get(self, relpath):
171
"""Return file-like object reading the contents of a remote file.
173
:see: Transport.get_bytes()/get_file()
175
return StringIO(self.get_bytes(relpath))
177
def get_bytes(self, relpath):
178
remote = self._remote_path(relpath)
179
protocol = SmartClientRequestProtocolOne(self._medium.get_request())
180
protocol.call('get', remote)
181
resp = protocol.read_response_tuple(True)
183
protocol.cancel_read_body()
184
self._translate_error(resp, relpath)
185
return protocol.read_body_bytes()
187
def _serialise_optional_mode(self, mode):
193
def mkdir(self, relpath, mode=None):
194
resp = self._call2('mkdir', self._remote_path(relpath),
195
self._serialise_optional_mode(mode))
196
self._translate_error(resp)
198
def put_bytes(self, relpath, upload_contents, mode=None):
199
# FIXME: upload_file is probably not safe for non-ascii characters -
200
# should probably just pass all parameters as length-delimited
202
resp = self._call_with_body_bytes('put',
203
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
205
self._translate_error(resp)
207
def put_bytes_non_atomic(self, relpath, bytes, mode=None,
208
create_parent_dir=False,
210
"""See Transport.put_bytes_non_atomic."""
211
# FIXME: no encoding in the transport!
212
create_parent_str = 'F'
213
if create_parent_dir:
214
create_parent_str = 'T'
216
resp = self._call_with_body_bytes(
218
(self._remote_path(relpath), self._serialise_optional_mode(mode),
219
create_parent_str, self._serialise_optional_mode(dir_mode)),
221
self._translate_error(resp)
223
def put_file(self, relpath, upload_file, mode=None):
224
# its not ideal to seek back, but currently put_non_atomic_file depends
225
# on transports not reading before failing - which is a faulty
226
# assumption I think - RBC 20060915
227
pos = upload_file.tell()
229
return self.put_bytes(relpath, upload_file.read(), mode)
231
upload_file.seek(pos)
234
def put_file_non_atomic(self, relpath, f, mode=None,
235
create_parent_dir=False,
237
return self.put_bytes_non_atomic(relpath, f.read(), mode=mode,
238
create_parent_dir=create_parent_dir,
241
def append_file(self, relpath, from_file, mode=None):
242
return self.append_bytes(relpath, from_file.read(), mode)
244
def append_bytes(self, relpath, bytes, mode=None):
245
resp = self._call_with_body_bytes(
247
(self._remote_path(relpath), self._serialise_optional_mode(mode)),
249
if resp[0] == 'appended':
251
self._translate_error(resp)
253
def delete(self, relpath):
254
resp = self._call2('delete', self._remote_path(relpath))
255
self._translate_error(resp)
257
def readv(self, relpath, offsets):
261
offsets = list(offsets)
263
sorted_offsets = sorted(offsets)
264
# turn the list of offsets into a stack
265
offset_stack = iter(offsets)
266
cur_offset_and_size = offset_stack.next()
267
coalesced = list(self._coalesce_offsets(sorted_offsets,
268
limit=self._max_readv_combine,
269
fudge_factor=self._bytes_to_read_before_seek))
271
protocol = SmartClientRequestProtocolOne(self._medium.get_request())
272
protocol.call_with_body_readv_array(
273
('readv', self._remote_path(relpath)),
274
[(c.start, c.length) for c in coalesced])
275
resp = protocol.read_response_tuple(True)
277
if resp[0] != 'readv':
278
# This should raise an exception
279
protocol.cancel_read_body()
280
self._translate_error(resp)
283
# FIXME: this should know how many bytes are needed, for clarity.
284
data = protocol.read_body_bytes()
285
# Cache the results, but only until they have been fulfilled
287
for c_offset in coalesced:
288
if len(data) < c_offset.length:
289
raise errors.ShortReadvError(relpath, c_offset.start,
290
c_offset.length, actual=len(data))
291
for suboffset, subsize in c_offset.ranges:
292
key = (c_offset.start+suboffset, subsize)
293
data_map[key] = data[suboffset:suboffset+subsize]
294
data = data[c_offset.length:]
296
# Now that we've read some data, see if we can yield anything back
297
while cur_offset_and_size in data_map:
298
this_data = data_map.pop(cur_offset_and_size)
299
yield cur_offset_and_size[0], this_data
300
cur_offset_and_size = offset_stack.next()
302
def rename(self, rel_from, rel_to):
304
self._remote_path(rel_from),
305
self._remote_path(rel_to))
307
def move(self, rel_from, rel_to):
309
self._remote_path(rel_from),
310
self._remote_path(rel_to))
312
def rmdir(self, relpath):
313
resp = self._call('rmdir', self._remote_path(relpath))
315
def _translate_error(self, resp, orig_path=None):
316
"""Raise an exception from a response"""
323
elif what == 'NoSuchFile':
324
if orig_path is not None:
325
error_path = orig_path
328
raise errors.NoSuchFile(error_path)
329
elif what == 'error':
330
raise errors.SmartProtocolError(unicode(resp[1]))
331
elif what == 'FileExists':
332
raise errors.FileExists(resp[1])
333
elif what == 'DirectoryNotEmpty':
334
raise errors.DirectoryNotEmpty(resp[1])
335
elif what == 'ShortReadvError':
336
raise errors.ShortReadvError(resp[1], int(resp[2]),
337
int(resp[3]), int(resp[4]))
338
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
339
encoding = str(resp[1]) # encoding must always be a string
343
reason = str(resp[5]) # reason must always be a string
344
if val.startswith('u:'):
345
val = val[2:].decode('utf-8')
346
elif val.startswith('s:'):
347
val = val[2:].decode('base64')
348
if what == 'UnicodeDecodeError':
349
raise UnicodeDecodeError(encoding, val, start, end, reason)
350
elif what == 'UnicodeEncodeError':
351
raise UnicodeEncodeError(encoding, val, start, end, reason)
352
elif what == "ReadOnlyError":
353
raise errors.TransportNotPossible('readonly transport')
355
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
357
def disconnect(self):
358
self._medium.disconnect()
360
def delete_tree(self, relpath):
361
raise errors.TransportNotPossible('readonly transport')
363
def stat(self, relpath):
364
resp = self._call2('stat', self._remote_path(relpath))
365
if resp[0] == 'stat':
366
return SmartStat(int(resp[1]), int(resp[2], 8))
368
self._translate_error(resp)
370
## def lock_read(self, relpath):
371
## """Lock the given file for shared (read) access.
372
## :return: A lock object, which should be passed to Transport.unlock()
374
## # The old RemoteBranch ignore lock for reading, so we will
375
## # continue that tradition and return a bogus lock object.
376
## class BogusLock(object):
377
## def __init__(self, path):
381
## return BogusLock(relpath)
386
def list_dir(self, relpath):
387
resp = self._call2('list_dir', self._remote_path(relpath))
388
if resp[0] == 'names':
389
return [name.encode('ascii') for name in resp[1:]]
391
self._translate_error(resp)
393
def iter_files_recursive(self):
394
resp = self._call2('iter_files_recursive', self._remote_path(''))
395
if resp[0] == 'names':
398
self._translate_error(resp)
402
class RemoteTCPTransport(RemoteTransport):
403
"""Connection to smart server over plain tcp.
405
This is essentially just a factory to get 'RemoteTransport(url,
406
SmartTCPClientMedium).
409
def __init__(self, url):
410
_scheme, _username, _password, _host, _port, _path = \
411
transport.split_url(url)
413
_port = BZR_DEFAULT_PORT
417
except (ValueError, TypeError), e:
418
raise errors.InvalidURL(
419
path=url, extra="invalid port %s" % _port)
420
medium = SmartTCPClientMedium(_host, _port)
421
super(RemoteTCPTransport, self).__init__(url, medium=medium)
424
class RemoteSSHTransport(RemoteTransport):
425
"""Connection to smart server over SSH.
427
This is essentially just a factory to get 'RemoteTransport(url,
428
SmartSSHClientMedium).
431
def __init__(self, url):
432
_scheme, _username, _password, _host, _port, _path = \
433
transport.split_url(url)
435
if _port is not None:
437
except (ValueError, TypeError), e:
438
raise errors.InvalidURL(path=url, extra="invalid port %s" %
440
medium = SmartSSHClientMedium(_host, _port, _username, _password)
441
super(RemoteSSHTransport, self).__init__(url, medium=medium)
444
class RemoteHTTPTransport(RemoteTransport):
445
"""Just a way to connect between a bzr+http:// url and http://.
447
This connection operates slightly differently than the RemoteSSHTransport.
448
It uses a plain http:// transport underneath, which defines what remote
449
.bzr/smart URL we are connected to. From there, all paths that are sent are
450
sent as relative paths, this way, the remote side can properly
451
de-reference them, since it is likely doing rewrite rules to translate an
452
HTTP path into a local path.
455
def __init__(self, url, http_transport=None):
456
assert url.startswith('bzr+http://')
458
if http_transport is None:
459
http_url = url[len('bzr+'):]
460
self._http_transport = transport.get_transport(http_url)
462
self._http_transport = http_transport
463
http_medium = self._http_transport.get_smart_medium()
464
super(RemoteHTTPTransport, self).__init__(url, medium=http_medium)
466
def _remote_path(self, relpath):
467
"""After connecting HTTP Transport only deals in relative URLs."""
473
def abspath(self, relpath):
474
"""Return the full url to the given relative path.
476
:param relpath: the relative path or path components
477
:type relpath: str or list
479
return self._unparse_url(self._combine_paths(self._path, relpath))
481
def clone(self, relative_url):
482
"""Make a new RemoteHTTPTransport related to me.
484
This is re-implemented rather than using the default
485
RemoteTransport.clone() because we must be careful about the underlying
489
abs_url = self.abspath(relative_url)
492
# By cloning the underlying http_transport, we are able to share the
494
new_transport = self._http_transport.clone(relative_url)
495
return RemoteHTTPTransport(abs_url, http_transport=new_transport)
498
def get_test_permutations():
499
"""Return (transport, server) permutations for testing."""
500
from bzrlib.smart import server
501
### We may need a little more test framework support to construct an
502
### appropriate RemoteTransport in the future.
503
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
