22
22
Media carry the bytes of the requests somehow (e.g. via TCP, wrapped in HTTP, or
23
23
over SSH), and pass them to and from the protocol logic. See the overview in
24
breezy/transport/smart/__init__.py.
24
bzrlib/transport/smart/__init__.py.
27
from __future__ import absolute_import
36
from ...lazy_import import lazy_import
31
from bzrlib.lazy_import import lazy_import
37
32
lazy_import(globals(), """
50
from breezy.i18n import gettext
51
from breezy.bzr.smart import client, protocol, request, signals, vfs
52
from breezy.transport import ssh
46
from bzrlib.smart import client, protocol, request, vfs
47
from bzrlib.transport import ssh
49
from bzrlib import osutils
59
51
# Throughout this module buffer size parameters are either limited to be at
60
52
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
133
114
def __init__(self):
134
115
self._push_back_buffer = None
136
def _push_back(self, data):
117
def _push_back(self, bytes):
137
118
"""Return unused bytes to the medium, because they belong to the next
140
121
This sets the _push_back_buffer to the given bytes.
142
if not isinstance(data, bytes):
143
raise TypeError(data)
144
123
if self._push_back_buffer is not None:
145
124
raise AssertionError(
146
125
"_push_back called when self._push_back_buffer is %r"
147
126
% (self._push_back_buffer,))
150
self._push_back_buffer = data
129
self._push_back_buffer = bytes
152
131
def _get_push_back_buffer(self):
153
if self._push_back_buffer == b'':
132
if self._push_back_buffer == '':
154
133
raise AssertionError(
155
134
'%s._push_back_buffer should never be the empty string, '
156
135
'which can be confused with EOF' % (self,))
249
214
while not self.finished:
250
215
server_protocol = self._build_protocol()
251
216
self._serve_one_request(server_protocol)
252
except errors.ConnectionTimeout as e:
253
trace.note('%s' % (e,))
254
trace.log_exception_quietly()
255
self._disconnect_client()
256
# We reported it, no reason to make a big fuss.
258
except Exception as e:
259
218
stderr.write("%s terminating on exception %s\n" % (self, e))
261
self._disconnect_client()
263
def _stop_gracefully(self):
264
"""When we finish this message, stop looking for more."""
265
trace.mutter('Stopping %s' % (self,))
268
def _disconnect_client(self):
269
"""Close the current connection. We stopped due to a timeout/etc."""
270
# The default implementation is a no-op, because that is all we used to
271
# do when disconnecting from a client. I suppose we never had the
272
# *server* initiate a disconnect, before
274
def _wait_for_bytes_with_timeout(self, timeout_seconds):
275
"""Wait for more bytes to be read, but timeout if none available.
277
This allows us to detect idle connections, and stop trying to read from
278
them, without setting the socket itself to non-blocking. This also
279
allows us to specify when we watch for idle timeouts.
281
:return: Did we timeout? (True if we timed out, False if there is data
284
raise NotImplementedError(self._wait_for_bytes_with_timeout)
286
221
def _build_protocol(self):
287
222
"""Identifies the version of the incoming request, and returns an
303
234
protocol.accept_bytes(unused_bytes)
306
def _wait_on_descriptor(self, fd, timeout_seconds):
307
"""select() on a file descriptor, waiting for nonblocking read()
309
This will raise a ConnectionTimeout exception if we do not get a
310
readable handle before timeout_seconds.
313
t_end = self._timer() + timeout_seconds
314
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
316
while not rs and not xs and self._timer() < t_end:
320
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
321
except (select.error, socket.error) as e:
322
err = getattr(e, 'errno', None)
323
if err is None and getattr(e, 'args', None) is not None:
324
# select.error doesn't have 'errno', it just has args[0]
326
if err in _bad_file_descriptor:
327
return # Not a socket indicates read() will fail
328
elif err == errno.EINTR:
329
# Interrupted, keep looping.
333
return # Socket may already be closed
336
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
337
% (timeout_seconds,))
339
237
def _serve_one_request(self, protocol):
340
238
"""Read one request from input, process, send back a response.
342
240
:param protocol: a SmartServerRequestProtocol.
347
243
self._serve_one_request_unguarded(protocol)
348
244
except KeyboardInterrupt:
350
except Exception as e:
351
247
self.terminate_due_to_error()
353
249
def terminate_due_to_error(self):
365
261
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
367
def __init__(self, sock, backing_transport, root_client_path='/',
263
def __init__(self, sock, backing_transport, root_client_path='/'):
371
266
:param sock: the socket the server will read from. It will be put
372
267
into blocking mode.
374
269
SmartServerStreamMedium.__init__(
375
self, backing_transport, root_client_path=root_client_path,
270
self, backing_transport, root_client_path=root_client_path)
377
271
sock.setblocking(True)
378
272
self.socket = sock
379
# Get the getpeername now, as we might be closed later when we care.
381
self._client_info = sock.getpeername()
383
self._client_info = '<unknown>'
386
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
389
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
392
274
def _serve_one_request_unguarded(self, protocol):
393
275
while protocol.next_read_size():
395
277
# than MAX_SOCKET_CHUNK ready, the socket will just return a
396
278
# short read immediately rather than block.
397
279
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
399
281
self.finished = True
401
283
protocol.accept_bytes(bytes)
403
285
self._push_back(protocol.unused_data)
405
def _disconnect_client(self):
406
"""Close the current connection. We stopped due to a timeout/etc."""
409
def _wait_for_bytes_with_timeout(self, timeout_seconds):
410
"""Wait for more bytes to be read, but timeout if none available.
412
This allows us to detect idle connections, and stop trying to read from
413
them, without setting the socket itself to non-blocking. This also
414
allows us to specify when we watch for idle timeouts.
416
:return: None, this will raise ConnectionTimeout if we time out before
419
return self._wait_on_descriptor(self.socket, timeout_seconds)
421
287
def _read_bytes(self, desired_count):
422
288
return osutils.read_bytes_from_socket(
423
289
self.socket, self._report_activity)
441
307
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
443
def __init__(self, in_file, out_file, backing_transport, timeout=None):
309
def __init__(self, in_file, out_file, backing_transport):
444
310
"""Construct new server.
446
312
:param in_file: Python file from which requests can be read.
447
313
:param out_file: Python file to write responses.
448
314
:param backing_transport: Transport for the directory served.
450
SmartServerStreamMedium.__init__(self, backing_transport,
316
SmartServerStreamMedium.__init__(self, backing_transport)
452
317
if sys.platform == 'win32':
453
318
# force binary mode for files
481
335
self._out.flush()
483
337
bytes = self.read_bytes(bytes_to_read)
485
339
# Connection has been closed.
486
340
self.finished = True
487
341
self._out.flush()
489
343
protocol.accept_bytes(bytes)
491
def _disconnect_client(self):
496
def _wait_for_bytes_with_timeout(self, timeout_seconds):
497
"""Wait for more bytes to be read, but timeout if none available.
499
This allows us to detect idle connections, and stop trying to read from
500
them, without setting the socket itself to non-blocking. This also
501
allows us to specify when we watch for idle timeouts.
503
:return: None, this will raise ConnectionTimeout if we time out before
506
if (getattr(self._in, 'fileno', None) is None
507
or sys.platform == 'win32'):
508
# You can't select() file descriptors on Windows.
511
return self._wait_on_descriptor(self._in, timeout_seconds)
512
except io.UnsupportedOperation:
515
345
def _read_bytes(self, desired_count):
516
346
return self._in.read(desired_count)
661
491
return self._medium._get_line()
664
class _VfsRefuser(object):
665
"""An object that refuses all VFS requests.
670
client._SmartClient.hooks.install_named_hook(
671
'call', self.check_vfs, 'vfs refuser')
673
def check_vfs(self, params):
675
request_method = request.request_handlers.get(params.method)
677
# A method we don't know about doesn't count as a VFS method.
679
if issubclass(request_method, vfs.VfsRequest):
680
raise HpssVfsRequestNotAllowed(params.method, params.args)
683
494
class _DebugCounter(object):
684
495
"""An object that counts the HPSS calls made to each client medium.
686
When a medium is garbage-collected, or failing that when
687
breezy.global_state exits, the total number of calls made on that medium
688
are reported via trace.note.
497
When a medium is garbage-collected, or failing that when atexit functions
498
are run, the total number of calls made on that medium are reported via
691
502
def __init__(self):
692
503
self.counts = weakref.WeakKeyDictionary()
693
504
client._SmartClient.hooks.install_named_hook(
694
505
'call', self.increment_call_count, 'hpss call counter')
695
breezy.get_global_state().cleanups.add_cleanup(self.flush_all)
506
atexit.register(self.flush_all)
697
508
def track(self, medium):
698
509
"""Start tracking calls made to a medium.
801
607
# which is newer than a previously supplied older-than version.
802
608
# This indicates that some smart verb call is not guarded
803
609
# appropriately (it should simply not have been tried).
610
raise AssertionError(
805
611
"_remember_remote_is_before(%r) called, but "
806
612
"_remember_remote_is_before(%r) was called previously."
807
, version_tuple, self._remote_version_is_before)
808
if 'hpss' in debug.debug_flags:
809
ui.ui_factory.show_warning(
810
"_remember_remote_is_before(%r) called, but "
811
"_remember_remote_is_before(%r) was called previously."
812
% (version_tuple, self._remote_version_is_before))
613
% (version_tuple, self._remote_version_is_before))
814
614
self._remote_version_is_before = version_tuple
816
616
def protocol_version(self):
905
705
return SmartClientStreamMediumRequest(self)
908
"""We have been disconnected, reset current state.
910
This resets things like _current_request and connected state.
913
self._current_request = None
916
708
class SmartSimplePipesClientMedium(SmartClientStreamMedium):
917
709
"""A client medium using simple pipes.
919
711
This client does not manage the pipes: it assumes they will always be open.
713
Note that if readable_pipe.read might raise IOError or OSError with errno
714
of EINTR, it must be safe to retry the read. Plain CPython fileobjects
715
(such as used for sys.stdin) are safe.
922
718
def __init__(self, readable_pipe, writeable_pipe, base):
924
720
self._readable_pipe = readable_pipe
925
721
self._writeable_pipe = writeable_pipe
927
def _accept_bytes(self, data):
723
def _accept_bytes(self, bytes):
928
724
"""See SmartClientStreamMedium.accept_bytes."""
930
self._writeable_pipe.write(data)
932
if e.errno in (errno.EINVAL, errno.EPIPE):
933
raise errors.ConnectionReset(
934
"Error trying to write to subprocess", e)
936
self._report_activity(len(data), 'write')
725
self._writeable_pipe.write(bytes)
726
self._report_activity(len(bytes), 'write')
938
728
def _flush(self):
939
729
"""See SmartClientStreamMedium._flush()."""
940
# Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
941
# However, testing shows that even when the child process is
942
# gone, this doesn't error.
943
730
self._writeable_pipe.flush()
945
732
def _read_bytes(self, count):
946
733
"""See SmartClientStreamMedium._read_bytes."""
947
bytes_to_read = min(count, _MAX_READ_SIZE)
948
data = self._readable_pipe.read(bytes_to_read)
949
self._report_activity(len(data), 'read')
953
class SSHParams(object):
954
"""A set of parameters for starting a remote bzr via SSH."""
734
bytes = osutils.until_no_eintr(self._readable_pipe.read, count)
735
self._report_activity(len(bytes), 'read')
739
class SmartSSHClientMedium(SmartClientStreamMedium):
740
"""A client medium using SSH."""
956
742
def __init__(self, host, port=None, username=None, password=None,
957
bzr_remote_path='bzr'):
960
self.username = username
961
self.password = password
962
self.bzr_remote_path = bzr_remote_path
965
class SmartSSHClientMedium(SmartClientStreamMedium):
966
"""A client medium using SSH.
968
It delegates IO to a SmartSimplePipesClientMedium or
969
SmartClientAlreadyConnectedSocketMedium (depending on platform).
972
def __init__(self, base, ssh_params, vendor=None):
743
base=None, vendor=None, bzr_remote_path=None):
973
744
"""Creates a client that will connect on the first use.
975
:param ssh_params: A SSHParams instance.
976
746
:param vendor: An optional override for the ssh vendor to use. See
977
breezy.transport.ssh for details on ssh vendors.
747
bzrlib.transport.ssh for details on ssh vendors.
979
self._real_medium = None
980
self._ssh_params = ssh_params
749
self._connected = False
751
self._password = password
753
self._username = username
981
754
# for the benefit of progress making a short description of this
983
756
self._scheme = 'bzr+ssh'
985
758
# _DebugCounter so we have to store all the values used in our repr
986
759
# method before calling the super init.
987
760
SmartClientStreamMedium.__init__(self, base)
761
self._read_from = None
762
self._ssh_connection = None
988
763
self._vendor = vendor
989
self._ssh_connection = None
764
self._write_to = None
765
self._bzr_remote_path = bzr_remote_path
991
767
def __repr__(self):
992
if self._ssh_params.port is None:
768
if self._port is None:
995
maybe_port = ':%s' % self._ssh_params.port
996
if self._ssh_params.username is None:
999
maybe_user = '%s@' % self._ssh_params.username
1000
return "%s(%s://%s%s%s/)" % (
771
maybe_port = ':%s' % self._port
772
return "%s(%s://%s@%s%s/)" % (
1001
773
self.__class__.__name__,
1004
self._ssh_params.host,
1007
779
def _accept_bytes(self, bytes):
1008
780
"""See SmartClientStreamMedium.accept_bytes."""
1009
781
self._ensure_connection()
1010
self._real_medium.accept_bytes(bytes)
782
self._write_to.write(bytes)
783
self._report_activity(len(bytes), 'write')
1012
785
def disconnect(self):
1013
786
"""See SmartClientMedium.disconnect()."""
1014
if self._real_medium is not None:
1015
self._real_medium.disconnect()
1016
self._real_medium = None
1017
if self._ssh_connection is not None:
1018
self._ssh_connection.close()
1019
self._ssh_connection = None
787
if not self._connected:
789
self._read_from.close()
790
self._write_to.close()
791
self._ssh_connection.close()
792
self._connected = False
1021
794
def _ensure_connection(self):
1022
795
"""Connect this medium if not already connected."""
1023
if self._real_medium is not None:
1025
798
if self._vendor is None:
1026
799
vendor = ssh._get_ssh_vendor()
1028
801
vendor = self._vendor
1029
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1030
self._ssh_params.password, self._ssh_params.host,
1031
self._ssh_params.port,
1032
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
802
self._ssh_connection = vendor.connect_ssh(self._username,
803
self._password, self._host, self._port,
804
command=[self._bzr_remote_path, 'serve', '--inet',
1033
805
'--directory=/', '--allow-writes'])
1034
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1035
if io_kind == 'socket':
1036
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1037
self.base, io_object)
1038
elif io_kind == 'pipes':
1039
read_from, write_to = io_object
1040
self._real_medium = SmartSimplePipesClientMedium(
1041
read_from, write_to, self.base)
1043
raise AssertionError(
1044
"Unexpected io_kind %r from %r"
1045
% (io_kind, self._ssh_connection))
1046
for hook in transport.Transport.hooks["post_connect"]:
806
self._read_from, self._write_to = \
807
self._ssh_connection.get_filelike_channels()
808
self._connected = True
1049
810
def _flush(self):
1050
811
"""See SmartClientStreamMedium._flush()."""
1051
self._real_medium._flush()
812
self._write_to.flush()
1053
814
def _read_bytes(self, count):
1054
815
"""See SmartClientStreamMedium.read_bytes."""
1055
if self._real_medium is None:
816
if not self._connected:
1056
817
raise errors.MediumNotConnected(self)
1057
return self._real_medium.read_bytes(count)
818
bytes_to_read = min(count, _MAX_READ_SIZE)
819
bytes = self._read_from.read(bytes_to_read)
820
self._report_activity(len(bytes), 'read')
1060
824
# Port 4155 is the default port for bzr://, registered with IANA.
1062
826
BZR_DEFAULT_PORT = 4155
1065
class SmartClientSocketMedium(SmartClientStreamMedium):
1066
"""A client medium using a socket.
1068
This class isn't usable directly. Use one of its subclasses instead.
1071
def __init__(self, base):
829
class SmartTCPClientMedium(SmartClientStreamMedium):
830
"""A client medium using TCP."""
832
def __init__(self, host, port, base):
833
"""Creates a client that will connect on the first use."""
1072
834
SmartClientStreamMedium.__init__(self, base)
835
self._connected = False
1073
838
self._socket = None
1074
self._connected = False
1076
840
def _accept_bytes(self, bytes):
1077
841
"""See SmartClientMedium.accept_bytes."""
1078
842
self._ensure_connection()
1079
843
osutils.send_all(self._socket, bytes, self._report_activity)
1081
def _ensure_connection(self):
1082
"""Connect this medium if not already connected."""
1083
raise NotImplementedError(self._ensure_connection)
1086
"""See SmartClientStreamMedium._flush().
1088
For sockets we do no flushing. For TCP sockets we may want to turn off
1089
TCP_NODELAY and add a means to do a flush, but that can be done in the
1093
def _read_bytes(self, count):
1094
"""See SmartClientMedium.read_bytes."""
1095
if not self._connected:
1096
raise errors.MediumNotConnected(self)
1097
return osutils.read_bytes_from_socket(
1098
self._socket, self._report_activity)
1100
845
def disconnect(self):
1101
846
"""See SmartClientMedium.disconnect()."""
1102
847
if not self._connected:
1127
862
sockaddrs = socket.getaddrinfo(self._host, port, socket.AF_UNSPEC,
1128
863
socket.SOCK_STREAM, 0, 0)
1129
except socket.gaierror as xxx_todo_changeme:
1130
(err_num, err_msg) = xxx_todo_changeme.args
864
except socket.gaierror, (err_num, err_msg):
1131
865
raise errors.ConnectionError("failed to lookup %s:%d: %s" %
1132
866
(self._host, port, err_msg))
1133
867
# Initialize err in case there are no addresses returned:
1134
last_err = socket.error("no address found for %s" % self._host)
868
err = socket.error("no address found for %s" % self._host)
1135
869
for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
1137
871
self._socket = socket.socket(family, socktype, proto)
1138
872
self._socket.setsockopt(socket.IPPROTO_TCP,
1139
873
socket.TCP_NODELAY, 1)
1140
874
self._socket.connect(sockaddr)
1141
except socket.error as err:
875
except socket.error, err:
1142
876
if self._socket is not None:
1143
877
self._socket.close()
1144
878
self._socket = None
1148
881
if self._socket is None:
1149
882
# socket errors either have a (string) or (errno, string) as their
1151
if isinstance(last_err.args, str):
1152
err_msg = last_err.args
884
if type(err.args) is str:
1154
err_msg = last_err.args[1]
887
err_msg = err.args[1]
1155
888
raise errors.ConnectionError("failed to connect to %s:%d: %s" %
1156
889
(self._host, port, err_msg))
1157
890
self._connected = True
1158
for hook in transport.Transport.hooks["post_connect"]:
1162
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1163
"""A client medium for an already connected socket.
1165
Note that this class will assume it "owns" the socket, so it will close it
1166
when its disconnect method is called.
1169
def __init__(self, base, sock):
1170
SmartClientSocketMedium.__init__(self, base)
1172
self._connected = True
1174
def _ensure_connection(self):
1175
# Already connected, by definition! So nothing to do.
893
"""See SmartClientStreamMedium._flush().
895
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
896
add a means to do a flush, but that can be done in the future.
899
def _read_bytes(self, count):
900
"""See SmartClientMedium.read_bytes."""
901
if not self._connected:
902
raise errors.MediumNotConnected(self)
903
return osutils.read_bytes_from_socket(
904
self._socket, self._report_activity)
1179
907
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
added
removed
bzrlib/smart/medium.py
