22
22
Media carry the bytes of the requests somehow (e.g. via TCP, wrapped in HTTP, or
23
23
over SSH), and pass them to and from the protocol logic. See the overview in
24
breezy/transport/smart/__init__.py.
24
bzrlib/transport/smart/__init__.py.
27
from __future__ import absolute_import
36
from ...lazy_import import lazy_import
33
from bzrlib.lazy_import import lazy_import
37
34
lazy_import(globals(), """
50
from breezy.i18n import gettext
51
from breezy.bzr.smart import client, protocol, request, signals, vfs
52
from breezy.transport import ssh
47
from bzrlib.smart import client, protocol, request, vfs
48
from bzrlib.transport import ssh
59
# Throughout this module buffer size parameters are either limited to be at
60
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
61
# For this module's purposes, MAX_SOCKET_CHUNK is a reasonable size for reads
62
# from non-sockets as well.
63
_MAX_READ_SIZE = osutils.MAX_SOCKET_CHUNK
66
class HpssVfsRequestNotAllowed(errors.BzrError):
68
_fmt = ("VFS requests over the smart server are not allowed. Encountered: "
69
"%(method)s, %(arguments)s.")
71
def __init__(self, method, arguments):
73
self.arguments = arguments
50
#usually already imported, and getting IllegalScoperReplacer on it here.
51
from bzrlib import osutils
53
# We must not read any more than 64k at a time so we don't risk "no buffer
54
# space available" errors on some platforms. Windows in particular is likely
55
# to give error 10053 or 10055 if we read more than 64k from a socket.
56
_MAX_READ_SIZE = 64 * 1024
76
59
def _get_protocol_factory_for_bytes(bytes):
249
216
while not self.finished:
250
217
server_protocol = self._build_protocol()
251
218
self._serve_one_request(server_protocol)
252
except errors.ConnectionTimeout as e:
253
trace.note('%s' % (e,))
254
trace.log_exception_quietly()
255
self._disconnect_client()
256
# We reported it, no reason to make a big fuss.
258
except Exception as e:
259
220
stderr.write("%s terminating on exception %s\n" % (self, e))
261
self._disconnect_client()
263
def _stop_gracefully(self):
264
"""When we finish this message, stop looking for more."""
265
trace.mutter('Stopping %s' % (self,))
268
def _disconnect_client(self):
269
"""Close the current connection. We stopped due to a timeout/etc."""
270
# The default implementation is a no-op, because that is all we used to
271
# do when disconnecting from a client. I suppose we never had the
272
# *server* initiate a disconnect, before
274
def _wait_for_bytes_with_timeout(self, timeout_seconds):
275
"""Wait for more bytes to be read, but timeout if none available.
277
This allows us to detect idle connections, and stop trying to read from
278
them, without setting the socket itself to non-blocking. This also
279
allows us to specify when we watch for idle timeouts.
281
:return: Did we timeout? (True if we timed out, False if there is data
284
raise NotImplementedError(self._wait_for_bytes_with_timeout)
286
223
def _build_protocol(self):
287
224
"""Identifies the version of the incoming request, and returns an
303
236
protocol.accept_bytes(unused_bytes)
306
def _wait_on_descriptor(self, fd, timeout_seconds):
307
"""select() on a file descriptor, waiting for nonblocking read()
309
This will raise a ConnectionTimeout exception if we do not get a
310
readable handle before timeout_seconds.
313
t_end = self._timer() + timeout_seconds
314
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
316
while not rs and not xs and self._timer() < t_end:
320
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
321
except (select.error, socket.error) as e:
322
err = getattr(e, 'errno', None)
323
if err is None and getattr(e, 'args', None) is not None:
324
# select.error doesn't have 'errno', it just has args[0]
326
if err in _bad_file_descriptor:
327
return # Not a socket indicates read() will fail
328
elif err == errno.EINTR:
329
# Interrupted, keep looping.
333
return # Socket may already be closed
336
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
337
% (timeout_seconds,))
339
239
def _serve_one_request(self, protocol):
340
240
"""Read one request from input, process, send back a response.
342
242
:param protocol: a SmartServerRequestProtocol.
347
245
self._serve_one_request_unguarded(protocol)
348
246
except KeyboardInterrupt:
350
except Exception as e:
351
249
self.terminate_due_to_error()
353
251
def terminate_due_to_error(self):
365
263
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
367
def __init__(self, sock, backing_transport, root_client_path='/',
265
def __init__(self, sock, backing_transport, root_client_path='/'):
371
268
:param sock: the socket the server will read from. It will be put
372
269
into blocking mode.
374
271
SmartServerStreamMedium.__init__(
375
self, backing_transport, root_client_path=root_client_path,
272
self, backing_transport, root_client_path=root_client_path)
377
273
sock.setblocking(True)
378
274
self.socket = sock
379
# Get the getpeername now, as we might be closed later when we care.
381
self._client_info = sock.getpeername()
383
self._client_info = '<unknown>'
386
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
389
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
392
276
def _serve_one_request_unguarded(self, protocol):
393
277
while protocol.next_read_size():
394
278
# We can safely try to read large chunks. If there is less data
395
# than MAX_SOCKET_CHUNK ready, the socket will just return a
396
# short read immediately rather than block.
397
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
279
# than _MAX_READ_SIZE ready, the socket wil just return a short
280
# read immediately rather than block.
281
bytes = self.read_bytes(_MAX_READ_SIZE)
399
283
self.finished = True
401
285
protocol.accept_bytes(bytes)
403
287
self._push_back(protocol.unused_data)
405
def _disconnect_client(self):
406
"""Close the current connection. We stopped due to a timeout/etc."""
409
def _wait_for_bytes_with_timeout(self, timeout_seconds):
410
"""Wait for more bytes to be read, but timeout if none available.
412
This allows us to detect idle connections, and stop trying to read from
413
them, without setting the socket itself to non-blocking. This also
414
allows us to specify when we watch for idle timeouts.
416
:return: None, this will raise ConnectionTimeout if we time out before
419
return self._wait_on_descriptor(self.socket, timeout_seconds)
421
289
def _read_bytes(self, desired_count):
422
return osutils.read_bytes_from_socket(
423
self.socket, self._report_activity)
290
return _read_bytes_from_socket(
291
self.socket.recv, desired_count, self._report_activity)
425
293
def terminate_due_to_error(self):
426
294
# TODO: This should log to a server log file, but no such thing
427
295
# exists yet. Andrew Bennetts 2006-09-29.
296
osutils.until_no_eintr(self.socket.close)
429
297
self.finished = True
431
299
def _write_out(self, bytes):
478
334
bytes_to_read = protocol.next_read_size()
479
335
if bytes_to_read == 0:
480
336
# Finished serving this request.
337
osutils.until_no_eintr(self._out.flush)
483
339
bytes = self.read_bytes(bytes_to_read)
485
341
# Connection has been closed.
486
342
self.finished = True
343
osutils.until_no_eintr(self._out.flush)
489
345
protocol.accept_bytes(bytes)
491
def _disconnect_client(self):
496
def _wait_for_bytes_with_timeout(self, timeout_seconds):
497
"""Wait for more bytes to be read, but timeout if none available.
499
This allows us to detect idle connections, and stop trying to read from
500
them, without setting the socket itself to non-blocking. This also
501
allows us to specify when we watch for idle timeouts.
503
:return: None, this will raise ConnectionTimeout if we time out before
506
if (getattr(self._in, 'fileno', None) is None
507
or sys.platform == 'win32'):
508
# You can't select() file descriptors on Windows.
511
return self._wait_on_descriptor(self._in, timeout_seconds)
512
except io.UnsupportedOperation:
515
347
def _read_bytes(self, desired_count):
516
return self._in.read(desired_count)
348
return osutils.until_no_eintr(self._in.read, desired_count)
518
350
def terminate_due_to_error(self):
519
351
# TODO: This should log to a server log file, but no such thing
520
352
# exists yet. Andrew Bennetts 2006-09-29.
353
osutils.until_no_eintr(self._out.close)
522
354
self.finished = True
524
356
def _write_out(self, bytes):
525
self._out.write(bytes)
357
osutils.until_no_eintr(self._out.write, bytes)
528
360
class SmartClientMediumRequest(object):
661
493
return self._medium._get_line()
664
class _VfsRefuser(object):
665
"""An object that refuses all VFS requests.
670
client._SmartClient.hooks.install_named_hook(
671
'call', self.check_vfs, 'vfs refuser')
673
def check_vfs(self, params):
675
request_method = request.request_handlers.get(params.method)
677
# A method we don't know about doesn't count as a VFS method.
679
if issubclass(request_method, vfs.VfsRequest):
680
raise HpssVfsRequestNotAllowed(params.method, params.args)
683
496
class _DebugCounter(object):
684
497
"""An object that counts the HPSS calls made to each client medium.
686
When a medium is garbage-collected, or failing that when
687
breezy.global_state exits, the total number of calls made on that medium
688
are reported via trace.note.
499
When a medium is garbage-collected, or failing that when atexit functions
500
are run, the total number of calls made on that medium are reported via
691
504
def __init__(self):
692
505
self.counts = weakref.WeakKeyDictionary()
693
506
client._SmartClient.hooks.install_named_hook(
694
507
'call', self.increment_call_count, 'hpss call counter')
695
breezy.get_global_state().cleanups.add_cleanup(self.flush_all)
508
atexit.register(self.flush_all)
697
510
def track(self, medium):
698
511
"""Start tracking calls made to a medium.
924
718
self._readable_pipe = readable_pipe
925
719
self._writeable_pipe = writeable_pipe
927
def _accept_bytes(self, data):
721
def _accept_bytes(self, bytes):
928
722
"""See SmartClientStreamMedium.accept_bytes."""
930
self._writeable_pipe.write(data)
932
if e.errno in (errno.EINVAL, errno.EPIPE):
933
raise errors.ConnectionReset(
934
"Error trying to write to subprocess", e)
936
self._report_activity(len(data), 'write')
723
osutils.until_no_eintr(self._writeable_pipe.write, bytes)
724
self._report_activity(len(bytes), 'write')
938
726
def _flush(self):
939
727
"""See SmartClientStreamMedium._flush()."""
940
# Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
941
# However, testing shows that even when the child process is
942
# gone, this doesn't error.
943
self._writeable_pipe.flush()
728
osutils.until_no_eintr(self._writeable_pipe.flush)
945
730
def _read_bytes(self, count):
946
731
"""See SmartClientStreamMedium._read_bytes."""
947
bytes_to_read = min(count, _MAX_READ_SIZE)
948
data = self._readable_pipe.read(bytes_to_read)
949
self._report_activity(len(data), 'read')
953
class SSHParams(object):
954
"""A set of parameters for starting a remote bzr via SSH."""
732
bytes = osutils.until_no_eintr(self._readable_pipe.read, count)
733
self._report_activity(len(bytes), 'read')
737
class SmartSSHClientMedium(SmartClientStreamMedium):
738
"""A client medium using SSH."""
956
740
def __init__(self, host, port=None, username=None, password=None,
957
bzr_remote_path='bzr'):
960
self.username = username
961
self.password = password
962
self.bzr_remote_path = bzr_remote_path
965
class SmartSSHClientMedium(SmartClientStreamMedium):
966
"""A client medium using SSH.
968
It delegates IO to a SmartSimplePipesClientMedium or
969
SmartClientAlreadyConnectedSocketMedium (depending on platform).
972
def __init__(self, base, ssh_params, vendor=None):
741
base=None, vendor=None, bzr_remote_path=None):
973
742
"""Creates a client that will connect on the first use.
975
:param ssh_params: A SSHParams instance.
976
744
:param vendor: An optional override for the ssh vendor to use. See
977
breezy.transport.ssh for details on ssh vendors.
745
bzrlib.transport.ssh for details on ssh vendors.
979
self._real_medium = None
980
self._ssh_params = ssh_params
981
# for the benefit of progress making a short description of this
983
self._scheme = 'bzr+ssh'
747
self._connected = False
749
self._password = password
751
self._username = username
984
752
# SmartClientStreamMedium stores the repr of this object in its
985
753
# _DebugCounter so we have to store all the values used in our repr
986
754
# method before calling the super init.
987
755
SmartClientStreamMedium.__init__(self, base)
756
self._read_from = None
757
self._ssh_connection = None
988
758
self._vendor = vendor
989
self._ssh_connection = None
759
self._write_to = None
760
self._bzr_remote_path = bzr_remote_path
761
# for the benefit of progress making a short description of this
763
self._scheme = 'bzr+ssh'
991
765
def __repr__(self):
992
if self._ssh_params.port is None:
995
maybe_port = ':%s' % self._ssh_params.port
996
if self._ssh_params.username is None:
999
maybe_user = '%s@' % self._ssh_params.username
1000
return "%s(%s://%s%s%s/)" % (
766
return "%s(connected=%r, username=%r, host=%r, port=%r)" % (
1001
767
self.__class__.__name__,
1004
self._ssh_params.host,
1007
773
def _accept_bytes(self, bytes):
1008
774
"""See SmartClientStreamMedium.accept_bytes."""
1009
775
self._ensure_connection()
1010
self._real_medium.accept_bytes(bytes)
776
osutils.until_no_eintr(self._write_to.write, bytes)
777
self._report_activity(len(bytes), 'write')
1012
779
def disconnect(self):
1013
780
"""See SmartClientMedium.disconnect()."""
1014
if self._real_medium is not None:
1015
self._real_medium.disconnect()
1016
self._real_medium = None
1017
if self._ssh_connection is not None:
1018
self._ssh_connection.close()
1019
self._ssh_connection = None
781
if not self._connected:
783
osutils.until_no_eintr(self._read_from.close)
784
osutils.until_no_eintr(self._write_to.close)
785
self._ssh_connection.close()
786
self._connected = False
1021
788
def _ensure_connection(self):
1022
789
"""Connect this medium if not already connected."""
1023
if self._real_medium is not None:
1025
792
if self._vendor is None:
1026
793
vendor = ssh._get_ssh_vendor()
1028
795
vendor = self._vendor
1029
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1030
self._ssh_params.password, self._ssh_params.host,
1031
self._ssh_params.port,
1032
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
796
self._ssh_connection = vendor.connect_ssh(self._username,
797
self._password, self._host, self._port,
798
command=[self._bzr_remote_path, 'serve', '--inet',
1033
799
'--directory=/', '--allow-writes'])
1034
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1035
if io_kind == 'socket':
1036
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1037
self.base, io_object)
1038
elif io_kind == 'pipes':
1039
read_from, write_to = io_object
1040
self._real_medium = SmartSimplePipesClientMedium(
1041
read_from, write_to, self.base)
1043
raise AssertionError(
1044
"Unexpected io_kind %r from %r"
1045
% (io_kind, self._ssh_connection))
1046
for hook in transport.Transport.hooks["post_connect"]:
800
self._read_from, self._write_to = \
801
self._ssh_connection.get_filelike_channels()
802
self._connected = True
1049
804
def _flush(self):
1050
805
"""See SmartClientStreamMedium._flush()."""
1051
self._real_medium._flush()
806
self._write_to.flush()
1053
808
def _read_bytes(self, count):
1054
809
"""See SmartClientStreamMedium.read_bytes."""
1055
if self._real_medium is None:
810
if not self._connected:
1056
811
raise errors.MediumNotConnected(self)
1057
return self._real_medium.read_bytes(count)
812
bytes_to_read = min(count, _MAX_READ_SIZE)
813
bytes = osutils.until_no_eintr(self._read_from.read, bytes_to_read)
814
self._report_activity(len(bytes), 'read')
1060
818
# Port 4155 is the default port for bzr://, registered with IANA.
1062
820
BZR_DEFAULT_PORT = 4155
1065
class SmartClientSocketMedium(SmartClientStreamMedium):
1066
"""A client medium using a socket.
1068
This class isn't usable directly. Use one of its subclasses instead.
1071
def __init__(self, base):
823
class SmartTCPClientMedium(SmartClientStreamMedium):
824
"""A client medium using TCP."""
826
def __init__(self, host, port, base):
827
"""Creates a client that will connect on the first use."""
1072
828
SmartClientStreamMedium.__init__(self, base)
829
self._connected = False
1073
832
self._socket = None
1074
self._connected = False
1076
834
def _accept_bytes(self, bytes):
1077
835
"""See SmartClientMedium.accept_bytes."""
1078
836
self._ensure_connection()
1079
837
osutils.send_all(self._socket, bytes, self._report_activity)
1081
def _ensure_connection(self):
1082
"""Connect this medium if not already connected."""
1083
raise NotImplementedError(self._ensure_connection)
1086
"""See SmartClientStreamMedium._flush().
1088
For sockets we do no flushing. For TCP sockets we may want to turn off
1089
TCP_NODELAY and add a means to do a flush, but that can be done in the
1093
def _read_bytes(self, count):
1094
"""See SmartClientMedium.read_bytes."""
1095
if not self._connected:
1096
raise errors.MediumNotConnected(self)
1097
return osutils.read_bytes_from_socket(
1098
self._socket, self._report_activity)
1100
839
def disconnect(self):
1101
840
"""See SmartClientMedium.disconnect()."""
1102
841
if not self._connected:
1104
self._socket.close()
843
osutils.until_no_eintr(self._socket.close)
1105
844
self._socket = None
1106
845
self._connected = False
1109
class SmartTCPClientMedium(SmartClientSocketMedium):
1110
"""A client medium that creates a TCP connection."""
1112
def __init__(self, host, port, base):
1113
"""Creates a client that will connect on the first use."""
1114
SmartClientSocketMedium.__init__(self, base)
1118
847
def _ensure_connection(self):
1119
848
"""Connect this medium if not already connected."""
1120
849
if self._connected:
1127
856
sockaddrs = socket.getaddrinfo(self._host, port, socket.AF_UNSPEC,
1128
857
socket.SOCK_STREAM, 0, 0)
1129
except socket.gaierror as xxx_todo_changeme:
1130
(err_num, err_msg) = xxx_todo_changeme.args
858
except socket.gaierror, (err_num, err_msg):
1131
859
raise errors.ConnectionError("failed to lookup %s:%d: %s" %
1132
860
(self._host, port, err_msg))
1133
861
# Initialize err in case there are no addresses returned:
1134
last_err = socket.error("no address found for %s" % self._host)
862
err = socket.error("no address found for %s" % self._host)
1135
863
for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
1137
865
self._socket = socket.socket(family, socktype, proto)
1138
866
self._socket.setsockopt(socket.IPPROTO_TCP,
1139
867
socket.TCP_NODELAY, 1)
1140
868
self._socket.connect(sockaddr)
1141
except socket.error as err:
869
except socket.error, err:
1142
870
if self._socket is not None:
1143
871
self._socket.close()
1144
872
self._socket = None
1148
875
if self._socket is None:
1149
876
# socket errors either have a (string) or (errno, string) as their
1151
if isinstance(last_err.args, str):
1152
err_msg = last_err.args
878
if type(err.args) is str:
1154
err_msg = last_err.args[1]
881
err_msg = err.args[1]
1155
882
raise errors.ConnectionError("failed to connect to %s:%d: %s" %
1156
883
(self._host, port, err_msg))
1157
884
self._connected = True
1158
for hook in transport.Transport.hooks["post_connect"]:
1162
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1163
"""A client medium for an already connected socket.
1165
Note that this class will assume it "owns" the socket, so it will close it
1166
when its disconnect method is called.
1169
def __init__(self, base, sock):
1170
SmartClientSocketMedium.__init__(self, base)
1172
self._connected = True
1174
def _ensure_connection(self):
1175
# Already connected, by definition! So nothing to do.
887
"""See SmartClientStreamMedium._flush().
889
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
890
add a means to do a flush, but that can be done in the future.
893
def _read_bytes(self, count):
894
"""See SmartClientMedium.read_bytes."""
895
if not self._connected:
896
raise errors.MediumNotConnected(self)
897
return _read_bytes_from_socket(
898
self._socket.recv, count, self._report_activity)
1179
901
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
added
removed
bzrlib/smart/medium.py
