83
85
one is being cloned from. Attributes such as the medium will
86
:param medium: The medium to use for this RemoteTransport. If None,
87
the medium from the _from_transport is shared. If both this
88
and _from_transport are None, a new medium will be built.
89
_from_transport and medium cannot both be specified.
88
:param medium: The medium to use for this RemoteTransport. This must be
89
supplied if _from_transport is None.
91
91
:param _client: Override the _SmartClient used by this transport. This
92
92
should only be used for testing purposes; normally this is
93
93
determined from the medium.
95
super(RemoteTransport, self).__init__(
96
url, _from_transport=_from_transport)
95
super(RemoteTransport, self).__init__(url,
96
_from_transport=_from_transport)
98
98
# The medium is the connection, except when we need to share it with
99
99
# other objects (RemoteBzrDir, RemoteRepository etc). In these cases
100
100
# what we want to share is really the shared connection.
102
if (_from_transport is not None
103
and isinstance(_from_transport, RemoteTransport)):
104
_client = _from_transport._client
105
elif _from_transport is None:
102
if _from_transport is None:
106
103
# If no _from_transport is specified, we need to intialize the
108
105
credentials = None
109
106
if medium is None:
110
107
medium, credentials = self._build_medium()
111
if 'hpss' in debug.debug_flags:
112
trace.mutter('hpss: Built a new medium: %s',
113
medium.__class__.__name__)
114
self._shared_connection = transport._SharedConnection(medium,
118
# No medium was specified, so share the medium from the
120
medium = self._shared_connection.connection
122
raise AssertionError(
123
"Both _from_transport (%r) and medium (%r) passed to "
124
"RemoteTransport.__init__, but these parameters are mutally "
125
"exclusive." % (_from_transport, medium))
108
self._shared_connection= transport._SharedConnection(medium,
127
111
if _client is None:
128
self._client = client._SmartClient(medium)
112
self._client = client._SmartClient(self.get_shared_medium())
130
114
self._client = _client
139
123
return None, None
141
def _report_activity(self, bytes, direction):
142
"""See Transport._report_activity.
144
Does nothing; the smart medium will report activity triggered by a
149
125
def is_readonly(self):
150
126
"""Smart server transport can do read/write file operations."""
152
resp = self._call2('Transport.is_readonly')
153
except errors.UnknownSmartMethod:
127
resp = self._call2('Transport.is_readonly')
128
if resp == ('yes', ):
130
elif resp == ('no', ):
132
elif (resp == ('error', "Generic bzr smart protocol error: "
133
"bad request 'Transport.is_readonly'") or
134
resp == ('error', "Generic bzr smart protocol error: "
135
"bad request u'Transport.is_readonly'")):
154
136
# XXX: nasty hack: servers before 0.16 don't have a
155
137
# 'Transport.is_readonly' verb, so we do what clients before 0.16
156
138
# did: assume False.
158
if resp == ('yes', ):
160
elif resp == ('no', ):
163
raise errors.UnexpectedSmartServerResponse(resp)
141
self._translate_error(resp)
142
raise errors.UnexpectedSmartServerResponse(resp)
165
144
def get_smart_client(self):
166
145
return self._get_connection()
168
147
def get_smart_medium(self):
169
148
return self._get_connection()
150
def get_shared_medium(self):
151
return self._get_shared_connection()
171
153
def _remote_path(self, relpath):
172
154
"""Returns the Unicode version of the absolute path for relpath."""
173
155
return self._combine_paths(self._path, relpath)
175
157
def _call(self, method, *args):
176
158
resp = self._call2(method, *args)
177
self._ensure_ok(resp)
159
self._translate_error(resp)
179
161
def _call2(self, method, *args):
180
162
"""Call a method on the remote server."""
182
return self._client.call(method, *args)
183
except errors.ErrorFromSmartServer, err:
184
# The first argument, if present, is always a path.
186
context = {'relpath': args[0]}
189
self._translate_error(err, **context)
163
return self._client.call(method, *args)
191
165
def _call_with_body_bytes(self, method, args, body):
192
166
"""Call a method on the remote server with body bytes."""
194
return self._client.call_with_body_bytes(method, args, body)
195
except errors.ErrorFromSmartServer, err:
196
# The first argument, if present, is always a path.
198
context = {'relpath': args[0]}
201
self._translate_error(err, **context)
167
return self._client.call_with_body_bytes(method, args, body)
203
169
def has(self, relpath):
204
170
"""Indicate whether a remote file of the given name exists or not.
211
177
elif resp == ('no', ):
214
raise errors.UnexpectedSmartServerResponse(resp)
180
self._translate_error(resp)
216
182
def get(self, relpath):
217
183
"""Return file-like object reading the contents of a remote file.
219
185
:see: Transport.get_bytes()/get_file()
221
187
return StringIO(self.get_bytes(relpath))
223
189
def get_bytes(self, relpath):
224
190
remote = self._remote_path(relpath)
226
resp, response_handler = self._client.call_expecting_body('get', remote)
227
except errors.ErrorFromSmartServer, err:
228
self._translate_error(err, relpath)
191
request = self.get_smart_medium().get_request()
192
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
193
smart_protocol.call('get', remote)
194
resp = smart_protocol.read_response_tuple(True)
229
195
if resp != ('ok', ):
230
response_handler.cancel_read_body()
231
raise errors.UnexpectedSmartServerResponse(resp)
232
return response_handler.read_body_bytes()
196
smart_protocol.cancel_read_body()
197
self._translate_error(resp, relpath)
198
return smart_protocol.read_body_bytes()
234
200
def _serialise_optional_mode(self, mode):
309
268
if resp[0] == 'appended':
310
269
return int(resp[1])
311
raise errors.UnexpectedSmartServerResponse(resp)
270
self._translate_error(resp)
313
272
def delete(self, relpath):
314
273
resp = self._call2('delete', self._remote_path(relpath))
315
self._ensure_ok(resp)
274
self._translate_error(resp)
317
276
def external_url(self):
318
277
"""See bzrlib.transport.Transport.external_url."""
319
278
# the external path for RemoteTransports is the base
322
def recommended_page_size(self):
323
"""Return the recommended page size for this transport."""
326
def _readv(self, relpath, offsets):
281
def readv(self, relpath, offsets):
330
285
offsets = list(offsets)
332
287
sorted_offsets = sorted(offsets)
288
# turn the list of offsets into a stack
289
offset_stack = iter(offsets)
290
cur_offset_and_size = offset_stack.next()
333
291
coalesced = list(self._coalesce_offsets(sorted_offsets,
334
292
limit=self._max_readv_combine,
335
fudge_factor=self._bytes_to_read_before_seek,
336
max_size=self._max_readv_bytes))
338
# now that we've coallesced things, avoid making enormous requests
343
if c.length + cur_len > self._max_readv_bytes:
344
requests.append(cur_request)
348
cur_request.append(c)
351
requests.append(cur_request)
352
if 'hpss' in debug.debug_flags:
353
trace.mutter('%s.readv %s offsets => %s coalesced'
354
' => %s requests (%s)',
355
self.__class__.__name__, len(offsets), len(coalesced),
356
len(requests), sum(map(len, requests)))
293
fudge_factor=self._bytes_to_read_before_seek))
295
request = self.get_smart_medium().get_request()
296
smart_protocol = protocol.SmartClientRequestProtocolOne(request)
297
smart_protocol.call_with_body_readv_array(
298
('readv', self._remote_path(relpath)),
299
[(c.start, c.length) for c in coalesced])
300
resp = smart_protocol.read_response_tuple(True)
302
if resp[0] != 'readv':
303
# This should raise an exception
304
smart_protocol.cancel_read_body()
305
self._translate_error(resp)
308
# FIXME: this should know how many bytes are needed, for clarity.
309
data = smart_protocol.read_body_bytes()
357
310
# Cache the results, but only until they have been fulfilled
359
# turn the list of offsets into a single stack to iterate
360
offset_stack = iter(offsets)
361
# using a list so it can be modified when passing down and coming back
362
next_offset = [offset_stack.next()]
363
for cur_request in requests:
365
result = self._client.call_with_body_readv_array(
366
('readv', self._remote_path(relpath),),
367
[(c.start, c.length) for c in cur_request])
368
resp, response_handler = result
369
except errors.ErrorFromSmartServer, err:
370
self._translate_error(err, relpath)
372
if resp[0] != 'readv':
373
# This should raise an exception
374
response_handler.cancel_read_body()
375
raise errors.UnexpectedSmartServerResponse(resp)
377
for res in self._handle_response(offset_stack, cur_request,
383
def _handle_response(self, offset_stack, coalesced, response_handler,
384
data_map, next_offset):
385
cur_offset_and_size = next_offset[0]
386
# FIXME: this should know how many bytes are needed, for clarity.
387
data = response_handler.read_body_bytes()
389
312
for c_offset in coalesced:
390
313
if len(data) < c_offset.length:
391
314
raise errors.ShortReadvError(relpath, c_offset.start,
392
315
c_offset.length, actual=len(data))
393
316
for suboffset, subsize in c_offset.ranges:
394
317
key = (c_offset.start+suboffset, subsize)
395
this_data = data[data_offset+suboffset:
396
data_offset+suboffset+subsize]
397
# Special case when the data is in-order, rather than packing
398
# into a map and then back out again. Benchmarking shows that
399
# this has 100% hit rate, but leave in the data_map work just
401
# TODO: Could we get away with using buffer() to avoid the
402
# memory copy? Callers would need to realize they may
403
# not have a real string.
404
if key == cur_offset_and_size:
405
yield cur_offset_and_size[0], this_data
406
cur_offset_and_size = next_offset[0] = offset_stack.next()
408
data_map[key] = this_data
409
data_offset += c_offset.length
318
data_map[key] = data[suboffset:suboffset+subsize]
319
data = data[c_offset.length:]
411
321
# Now that we've read some data, see if we can yield anything back
412
322
while cur_offset_and_size in data_map:
413
323
this_data = data_map.pop(cur_offset_and_size)
414
324
yield cur_offset_and_size[0], this_data
415
cur_offset_and_size = next_offset[0] = offset_stack.next()
325
cur_offset_and_size = offset_stack.next()
417
327
def rename(self, rel_from, rel_to):
418
328
self._call('rename',
427
337
def rmdir(self, relpath):
428
338
resp = self._call('rmdir', self._remote_path(relpath))
430
def _ensure_ok(self, resp):
432
raise errors.UnexpectedSmartServerResponse(resp)
434
def _translate_error(self, err, relpath=None):
435
remote._translate_error(err, path=relpath)
340
def _translate_error(self, resp, orig_path=None):
341
"""Raise an exception from a response"""
348
elif what == 'NoSuchFile':
349
if orig_path is not None:
350
error_path = orig_path
353
raise errors.NoSuchFile(error_path)
354
elif what == 'error':
355
raise errors.SmartProtocolError(unicode(resp[1]))
356
elif what == 'FileExists':
357
raise errors.FileExists(resp[1])
358
elif what == 'DirectoryNotEmpty':
359
raise errors.DirectoryNotEmpty(resp[1])
360
elif what == 'ShortReadvError':
361
raise errors.ShortReadvError(resp[1], int(resp[2]),
362
int(resp[3]), int(resp[4]))
363
elif what in ('UnicodeEncodeError', 'UnicodeDecodeError'):
364
encoding = str(resp[1]) # encoding must always be a string
368
reason = str(resp[5]) # reason must always be a string
369
if val.startswith('u:'):
370
val = val[2:].decode('utf-8')
371
elif val.startswith('s:'):
372
val = val[2:].decode('base64')
373
if what == 'UnicodeDecodeError':
374
raise UnicodeDecodeError(encoding, val, start, end, reason)
375
elif what == 'UnicodeEncodeError':
376
raise UnicodeEncodeError(encoding, val, start, end, reason)
377
elif what == "ReadOnlyError":
378
raise errors.TransportNotPossible('readonly transport')
379
elif what == "ReadError":
380
if orig_path is not None:
381
error_path = orig_path
384
raise errors.ReadError(error_path)
386
raise errors.SmartProtocolError('unexpected smart server error: %r' % (resp,))
437
388
def disconnect(self):
438
389
self.get_smart_medium().disconnect()
391
def delete_tree(self, relpath):
392
raise errors.TransportNotPossible('readonly transport')
440
394
def stat(self, relpath):
441
395
resp = self._call2('stat', self._remote_path(relpath))
442
396
if resp[0] == 'stat':
443
397
return _SmartStat(int(resp[1]), int(resp[2], 8))
444
raise errors.UnexpectedSmartServerResponse(resp)
399
self._translate_error(resp)
446
401
## def lock_read(self, relpath):
447
402
## """Lock the given file for shared (read) access.
463
418
resp = self._call2('list_dir', self._remote_path(relpath))
464
419
if resp[0] == 'names':
465
420
return [name.encode('ascii') for name in resp[1:]]
466
raise errors.UnexpectedSmartServerResponse(resp)
422
self._translate_error(resp)
468
424
def iter_files_recursive(self):
469
425
resp = self._call2('iter_files_recursive', self._remote_path(''))
470
426
if resp[0] == 'names':
472
raise errors.UnexpectedSmartServerResponse(resp)
429
self._translate_error(resp)
475
432
class RemoteTCPTransport(RemoteTransport):
476
433
"""Connection to smart server over plain tcp.
478
435
This is essentially just a factory to get 'RemoteTransport(url,
479
436
SmartTCPClientMedium).
482
439
def _build_medium(self):
483
client_medium = medium.SmartTCPClientMedium(
484
self._host, self._port, self.base)
485
return client_medium, None
488
class RemoteTCPTransportV2Only(RemoteTransport):
489
"""Connection to smart server over plain tcp with the client hard-coded to
490
assume protocol v2 and remote server version <= 1.6.
492
This should only be used for testing.
495
def _build_medium(self):
496
client_medium = medium.SmartTCPClientMedium(
497
self._host, self._port, self.base)
498
client_medium._protocol_version = 2
499
client_medium._remember_remote_is_before((1, 6))
500
return client_medium, None
440
assert self.base.startswith('bzr://')
441
if self._port is None:
442
self._port = BZR_DEFAULT_PORT
443
return medium.SmartTCPClientMedium(self._host, self._port), None
503
446
class RemoteSSHTransport(RemoteTransport):
510
453
def _build_medium(self):
511
location_config = config.LocationConfig(self.base)
512
bzr_remote_path = location_config.get_bzr_remote_path()
515
auth = config.AuthenticationConfig()
516
user = auth.get_user('ssh', self._host, self._port)
517
client_medium = medium.SmartSSHClientMedium(self._host, self._port,
518
user, self._password, self.base,
519
bzr_remote_path=bzr_remote_path)
520
return client_medium, (user, self._password)
454
assert self.base.startswith('bzr+ssh://')
455
# ssh will prompt the user for a password if needed and if none is
456
# provided but it will not give it back, so no credentials can be
458
return medium.SmartSSHClientMedium(self._host, self._port,
459
self._user, self._password), None
523
462
class RemoteHTTPTransport(RemoteTransport):
524
463
"""Just a way to connect between a bzr+http:// url and http://.
526
465
This connection operates slightly differently than the RemoteSSHTransport.
527
466
It uses a plain http:// transport underneath, which defines what remote
528
467
.bzr/smart URL we are connected to. From there, all paths that are sent are
568
509
smart requests may be different). This is so that the server doesn't
569
510
have to handle .bzr/smart requests at arbitrary places inside .bzr
570
511
directories, just at the initial URL the user uses.
513
The exception is parent paths (i.e. relative_url of "..").
573
516
abs_url = self.abspath(relative_url)
575
518
abs_url = self.base
519
# We either use the exact same http_transport (for child locations), or
520
# a clone of the underlying http_transport (for parent locations). This
521
# means we share the connection.
522
norm_base = urlutils.normalize_url(self.base)
523
norm_abs_url = urlutils.normalize_url(abs_url)
524
normalized_rel_url = urlutils.relative_url(norm_base, norm_abs_url)
525
if normalized_rel_url == ".." or normalized_rel_url.startswith("../"):
526
http_transport = self._http_transport.clone(normalized_rel_url)
528
http_transport = self._http_transport
576
529
return RemoteHTTPTransport(abs_url,
577
530
_from_transport=self,
578
http_transport=self._http_transport)
580
def _redirected_to(self, source, target):
581
"""See transport._redirected_to"""
582
redirected = self._http_transport._redirected_to(source, target)
583
if (redirected is not None
584
and isinstance(redirected, type(self._http_transport))):
585
return RemoteHTTPTransport('bzr+' + redirected.external_url(),
586
http_transport=redirected)
588
# Either None or a transport for a different protocol
592
class HintingSSHTransport(transport.Transport):
593
"""Simple transport that handles ssh:// and points out bzr+ssh://."""
595
def __init__(self, url):
596
raise errors.UnsupportedProtocol(url,
597
'bzr supports bzr+ssh to operate over ssh, use "bzr+%s".' % url)
531
http_transport=http_transport)
600
534
def get_test_permutations():
601
535
"""Return (transport, server) permutations for testing."""
602
536
### We may need a little more test framework support to construct an
603
537
### appropriate RemoteTransport in the future.
604
from bzrlib.tests import test_server
605
return [(RemoteTCPTransport, test_server.SmartTCPServer_for_testing)]
538
from bzrlib.smart import server
539
return [(RemoteTCPTransport, server.SmartTCPServer_for_testing)]
added
removed
bzrlib/transport/remote.py
