22
22
Media carry the bytes of the requests somehow (e.g. via TCP, wrapped in HTTP, or
23
23
over SSH), and pass them to and from the protocol logic. See the overview in
24
breezy/transport/smart/__init__.py.
24
bzrlib/transport/smart/__init__.py.
27
from __future__ import absolute_import
38
import thread as _thread
41
from ...lazy_import import lazy_import
31
from bzrlib.lazy_import import lazy_import
42
32
lazy_import(globals(), """
54
from breezy.i18n import gettext
55
from breezy.bzr.smart import client, protocol, request, signals, vfs
56
from breezy.transport import ssh
46
from bzrlib.smart import client, protocol, request, vfs
47
from bzrlib.transport import ssh
49
from bzrlib import osutils
63
51
# Throughout this module buffer size parameters are either limited to be at
64
52
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
137
114
def __init__(self):
138
115
self._push_back_buffer = None
140
def _push_back(self, data):
117
def _push_back(self, bytes):
141
118
"""Return unused bytes to the medium, because they belong to the next
144
121
This sets the _push_back_buffer to the given bytes.
146
if not isinstance(data, bytes):
147
raise TypeError(data)
148
123
if self._push_back_buffer is not None:
149
124
raise AssertionError(
150
125
"_push_back called when self._push_back_buffer is %r"
151
126
% (self._push_back_buffer,))
154
self._push_back_buffer = data
129
self._push_back_buffer = bytes
156
131
def _get_push_back_buffer(self):
157
if self._push_back_buffer == b'':
132
if self._push_back_buffer == '':
158
133
raise AssertionError(
159
134
'%s._push_back_buffer should never be the empty string, '
160
135
'which can be confused with EOF' % (self,))
253
214
while not self.finished:
254
215
server_protocol = self._build_protocol()
255
216
self._serve_one_request(server_protocol)
256
except errors.ConnectionTimeout as e:
257
trace.note('%s' % (e,))
258
trace.log_exception_quietly()
259
self._disconnect_client()
260
# We reported it, no reason to make a big fuss.
262
except Exception as e:
263
218
stderr.write("%s terminating on exception %s\n" % (self, e))
265
self._disconnect_client()
267
def _stop_gracefully(self):
268
"""When we finish this message, stop looking for more."""
269
trace.mutter('Stopping %s' % (self,))
272
def _disconnect_client(self):
273
"""Close the current connection. We stopped due to a timeout/etc."""
274
# The default implementation is a no-op, because that is all we used to
275
# do when disconnecting from a client. I suppose we never had the
276
# *server* initiate a disconnect, before
278
def _wait_for_bytes_with_timeout(self, timeout_seconds):
279
"""Wait for more bytes to be read, but timeout if none available.
281
This allows us to detect idle connections, and stop trying to read from
282
them, without setting the socket itself to non-blocking. This also
283
allows us to specify when we watch for idle timeouts.
285
:return: Did we timeout? (True if we timed out, False if there is data
288
raise NotImplementedError(self._wait_for_bytes_with_timeout)
290
221
def _build_protocol(self):
291
222
"""Identifies the version of the incoming request, and returns an
307
234
protocol.accept_bytes(unused_bytes)
310
def _wait_on_descriptor(self, fd, timeout_seconds):
311
"""select() on a file descriptor, waiting for nonblocking read()
313
This will raise a ConnectionTimeout exception if we do not get a
314
readable handle before timeout_seconds.
317
t_end = self._timer() + timeout_seconds
318
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
320
while not rs and not xs and self._timer() < t_end:
324
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
325
except (select.error, socket.error) as e:
326
err = getattr(e, 'errno', None)
327
if err is None and getattr(e, 'args', None) is not None:
328
# select.error doesn't have 'errno', it just has args[0]
330
if err in _bad_file_descriptor:
331
return # Not a socket indicates read() will fail
332
elif err == errno.EINTR:
333
# Interrupted, keep looping.
337
return # Socket may already be closed
340
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
341
% (timeout_seconds,))
343
237
def _serve_one_request(self, protocol):
344
238
"""Read one request from input, process, send back a response.
346
240
:param protocol: a SmartServerRequestProtocol.
351
243
self._serve_one_request_unguarded(protocol)
352
244
except KeyboardInterrupt:
354
except Exception as e:
355
247
self.terminate_due_to_error()
357
249
def terminate_due_to_error(self):
369
261
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
371
def __init__(self, sock, backing_transport, root_client_path='/',
263
def __init__(self, sock, backing_transport, root_client_path='/'):
375
266
:param sock: the socket the server will read from. It will be put
376
267
into blocking mode.
378
269
SmartServerStreamMedium.__init__(
379
self, backing_transport, root_client_path=root_client_path,
270
self, backing_transport, root_client_path=root_client_path)
381
271
sock.setblocking(True)
382
272
self.socket = sock
383
# Get the getpeername now, as we might be closed later when we care.
385
self._client_info = sock.getpeername()
387
self._client_info = '<unknown>'
390
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
393
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
396
274
def _serve_one_request_unguarded(self, protocol):
397
275
while protocol.next_read_size():
399
277
# than MAX_SOCKET_CHUNK ready, the socket will just return a
400
278
# short read immediately rather than block.
401
279
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
403
281
self.finished = True
405
283
protocol.accept_bytes(bytes)
407
285
self._push_back(protocol.unused_data)
409
def _disconnect_client(self):
410
"""Close the current connection. We stopped due to a timeout/etc."""
413
def _wait_for_bytes_with_timeout(self, timeout_seconds):
414
"""Wait for more bytes to be read, but timeout if none available.
416
This allows us to detect idle connections, and stop trying to read from
417
them, without setting the socket itself to non-blocking. This also
418
allows us to specify when we watch for idle timeouts.
420
:return: None, this will raise ConnectionTimeout if we time out before
423
return self._wait_on_descriptor(self.socket, timeout_seconds)
425
287
def _read_bytes(self, desired_count):
426
288
return osutils.read_bytes_from_socket(
427
289
self.socket, self._report_activity)
433
295
self.finished = True
435
297
def _write_out(self, bytes):
436
tstart = osutils.perf_counter()
298
tstart = osutils.timer_func()
437
299
osutils.send_all(self.socket, bytes, self._report_activity)
438
300
if 'hpss' in debug.debug_flags:
439
thread_id = _thread.get_ident()
301
thread_id = thread.get_ident()
440
302
trace.mutter('%12s: [%s] %d bytes to the socket in %.3fs'
441
303
% ('wrote', thread_id, len(bytes),
442
osutils.perf_counter() - tstart))
304
osutils.timer_func() - tstart))
445
307
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
447
def __init__(self, in_file, out_file, backing_transport, timeout=None):
309
def __init__(self, in_file, out_file, backing_transport):
448
310
"""Construct new server.
450
312
:param in_file: Python file from which requests can be read.
451
313
:param out_file: Python file to write responses.
452
314
:param backing_transport: Transport for the directory served.
454
SmartServerStreamMedium.__init__(self, backing_transport,
316
SmartServerStreamMedium.__init__(self, backing_transport)
456
317
if sys.platform == 'win32':
457
318
# force binary mode for files
485
335
self._out.flush()
487
337
bytes = self.read_bytes(bytes_to_read)
489
339
# Connection has been closed.
490
340
self.finished = True
491
341
self._out.flush()
493
343
protocol.accept_bytes(bytes)
495
def _disconnect_client(self):
500
def _wait_for_bytes_with_timeout(self, timeout_seconds):
501
"""Wait for more bytes to be read, but timeout if none available.
503
This allows us to detect idle connections, and stop trying to read from
504
them, without setting the socket itself to non-blocking. This also
505
allows us to specify when we watch for idle timeouts.
507
:return: None, this will raise ConnectionTimeout if we time out before
510
if (getattr(self._in, 'fileno', None) is None
511
or sys.platform == 'win32'):
512
# You can't select() file descriptors on Windows.
515
return self._wait_on_descriptor(self._in, timeout_seconds)
516
except io.UnsupportedOperation:
519
345
def _read_bytes(self, desired_count):
520
346
return self._in.read(desired_count)
665
491
return self._medium._get_line()
668
class _VfsRefuser(object):
669
"""An object that refuses all VFS requests.
674
client._SmartClient.hooks.install_named_hook(
675
'call', self.check_vfs, 'vfs refuser')
677
def check_vfs(self, params):
679
request_method = request.request_handlers.get(params.method)
681
# A method we don't know about doesn't count as a VFS method.
683
if issubclass(request_method, vfs.VfsRequest):
684
raise HpssVfsRequestNotAllowed(params.method, params.args)
687
494
class _DebugCounter(object):
688
495
"""An object that counts the HPSS calls made to each client medium.
690
When a medium is garbage-collected, or failing that when
691
breezy.global_state exits, the total number of calls made on that medium
692
are reported via trace.note.
497
When a medium is garbage-collected, or failing that when atexit functions
498
are run, the total number of calls made on that medium are reported via
695
502
def __init__(self):
696
503
self.counts = weakref.WeakKeyDictionary()
697
504
client._SmartClient.hooks.install_named_hook(
698
505
'call', self.increment_call_count, 'hpss call counter')
699
breezy.get_global_state().exit_stack.callback(self.flush_all)
506
atexit.register(self.flush_all)
701
508
def track(self, medium):
702
509
"""Start tracking calls made to a medium.
801
602
:seealso: _is_remote_before
803
604
if (self._remote_version_is_before is not None and
804
version_tuple > self._remote_version_is_before):
605
version_tuple > self._remote_version_is_before):
805
606
# We have been told that the remote side is older than some version
806
607
# which is newer than a previously supplied older-than version.
807
608
# This indicates that some smart verb call is not guarded
808
609
# appropriately (it should simply not have been tried).
810
611
"_remember_remote_is_before(%r) called, but "
811
"_remember_remote_is_before(%r) was called previously.", version_tuple, self._remote_version_is_before)
612
"_remember_remote_is_before(%r) was called previously."
613
, version_tuple, self._remote_version_is_before)
812
614
if 'hpss' in debug.debug_flags:
813
615
ui.ui_factory.show_warning(
814
616
"_remember_remote_is_before(%r) called, but "
826
628
medium_request = self.get_request()
827
629
# Send a 'hello' request in protocol version one, for maximum
828
630
# backwards compatibility.
829
client_protocol = protocol.SmartClientRequestProtocolOne(
631
client_protocol = protocol.SmartClientRequestProtocolOne(medium_request)
831
632
client_protocol.query_version()
832
633
self._done_hello = True
833
except errors.SmartProtocolError as e:
634
except errors.SmartProtocolError, e:
834
635
# Cache the error, just like we would cache a successful
836
637
self._protocol_version_error = e
910
711
return SmartClientStreamMediumRequest(self)
913
"""We have been disconnected, reset current state.
915
This resets things like _current_request and connected state.
918
self._current_request = None
921
714
class SmartSimplePipesClientMedium(SmartClientStreamMedium):
922
715
"""A client medium using simple pipes.
924
717
This client does not manage the pipes: it assumes they will always be open.
719
Note that if readable_pipe.read might raise IOError or OSError with errno
720
of EINTR, it must be safe to retry the read. Plain CPython fileobjects
721
(such as used for sys.stdin) are safe.
927
724
def __init__(self, readable_pipe, writeable_pipe, base):
929
726
self._readable_pipe = readable_pipe
930
727
self._writeable_pipe = writeable_pipe
932
def _accept_bytes(self, data):
729
def _accept_bytes(self, bytes):
933
730
"""See SmartClientStreamMedium.accept_bytes."""
935
self._writeable_pipe.write(data)
937
if e.errno in (errno.EINVAL, errno.EPIPE):
938
raise errors.ConnectionReset(
939
"Error trying to write to subprocess", e)
941
self._report_activity(len(data), 'write')
731
self._writeable_pipe.write(bytes)
732
self._report_activity(len(bytes), 'write')
943
734
def _flush(self):
944
735
"""See SmartClientStreamMedium._flush()."""
945
# Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
946
# However, testing shows that even when the child process is
947
# gone, this doesn't error.
948
736
self._writeable_pipe.flush()
950
738
def _read_bytes(self, count):
951
739
"""See SmartClientStreamMedium._read_bytes."""
952
bytes_to_read = min(count, _MAX_READ_SIZE)
953
data = self._readable_pipe.read(bytes_to_read)
954
self._report_activity(len(data), 'read')
958
class SSHParams(object):
959
"""A set of parameters for starting a remote bzr via SSH."""
740
bytes = osutils.until_no_eintr(self._readable_pipe.read, count)
741
self._report_activity(len(bytes), 'read')
745
class SmartSSHClientMedium(SmartClientStreamMedium):
746
"""A client medium using SSH."""
961
748
def __init__(self, host, port=None, username=None, password=None,
962
bzr_remote_path='bzr'):
965
self.username = username
966
self.password = password
967
self.bzr_remote_path = bzr_remote_path
970
class SmartSSHClientMedium(SmartClientStreamMedium):
971
"""A client medium using SSH.
973
It delegates IO to a SmartSimplePipesClientMedium or
974
SmartClientAlreadyConnectedSocketMedium (depending on platform).
977
def __init__(self, base, ssh_params, vendor=None):
749
base=None, vendor=None, bzr_remote_path=None):
978
750
"""Creates a client that will connect on the first use.
980
:param ssh_params: A SSHParams instance.
981
752
:param vendor: An optional override for the ssh vendor to use. See
982
breezy.transport.ssh for details on ssh vendors.
753
bzrlib.transport.ssh for details on ssh vendors.
984
self._real_medium = None
985
self._ssh_params = ssh_params
755
self._connected = False
757
self._password = password
759
self._username = username
986
760
# for the benefit of progress making a short description of this
988
762
self._scheme = 'bzr+ssh'
990
764
# _DebugCounter so we have to store all the values used in our repr
991
765
# method before calling the super init.
992
766
SmartClientStreamMedium.__init__(self, base)
767
self._read_from = None
768
self._ssh_connection = None
993
769
self._vendor = vendor
994
self._ssh_connection = None
770
self._write_to = None
771
self._bzr_remote_path = bzr_remote_path
996
773
def __repr__(self):
997
if self._ssh_params.port is None:
774
if self._port is None:
1000
maybe_port = ':%s' % self._ssh_params.port
1001
if self._ssh_params.username is None:
1004
maybe_user = '%s@' % self._ssh_params.username
1005
return "%s(%s://%s%s%s/)" % (
777
maybe_port = ':%s' % self._port
778
return "%s(%s://%s@%s%s/)" % (
1006
779
self.__class__.__name__,
1009
self._ssh_params.host,
1012
785
def _accept_bytes(self, bytes):
1013
786
"""See SmartClientStreamMedium.accept_bytes."""
1014
787
self._ensure_connection()
1015
self._real_medium.accept_bytes(bytes)
788
self._write_to.write(bytes)
789
self._report_activity(len(bytes), 'write')
1017
791
def disconnect(self):
1018
792
"""See SmartClientMedium.disconnect()."""
1019
if self._real_medium is not None:
1020
self._real_medium.disconnect()
1021
self._real_medium = None
1022
if self._ssh_connection is not None:
1023
self._ssh_connection.close()
1024
self._ssh_connection = None
793
if not self._connected:
795
self._read_from.close()
796
self._write_to.close()
797
self._ssh_connection.close()
798
self._connected = False
1026
800
def _ensure_connection(self):
1027
801
"""Connect this medium if not already connected."""
1028
if self._real_medium is not None:
1030
804
if self._vendor is None:
1031
805
vendor = ssh._get_ssh_vendor()
1033
807
vendor = self._vendor
1034
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1035
self._ssh_params.password, self._ssh_params.host,
1036
self._ssh_params.port,
1037
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
1038
'--directory=/', '--allow-writes'])
1039
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1040
if io_kind == 'socket':
1041
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1042
self.base, io_object)
1043
elif io_kind == 'pipes':
1044
read_from, write_to = io_object
1045
self._real_medium = SmartSimplePipesClientMedium(
1046
read_from, write_to, self.base)
1048
raise AssertionError(
1049
"Unexpected io_kind %r from %r"
1050
% (io_kind, self._ssh_connection))
1051
for hook in transport.Transport.hooks["post_connect"]:
808
self._ssh_connection = vendor.connect_ssh(self._username,
809
self._password, self._host, self._port,
810
command=[self._bzr_remote_path, 'serve', '--inet',
811
'--directory=/', '--allow-writes'])
812
self._read_from, self._write_to = \
813
self._ssh_connection.get_filelike_channels()
814
self._connected = True
1054
816
def _flush(self):
1055
817
"""See SmartClientStreamMedium._flush()."""
1056
self._real_medium._flush()
818
self._write_to.flush()
1058
820
def _read_bytes(self, count):
1059
821
"""See SmartClientStreamMedium.read_bytes."""
1060
if self._real_medium is None:
822
if not self._connected:
1061
823
raise errors.MediumNotConnected(self)
1062
return self._real_medium.read_bytes(count)
824
bytes_to_read = min(count, _MAX_READ_SIZE)
825
bytes = self._read_from.read(bytes_to_read)
826
self._report_activity(len(bytes), 'read')
1065
830
# Port 4155 is the default port for bzr://, registered with IANA.
1067
832
BZR_DEFAULT_PORT = 4155
1070
class SmartClientSocketMedium(SmartClientStreamMedium):
1071
"""A client medium using a socket.
1073
This class isn't usable directly. Use one of its subclasses instead.
1076
def __init__(self, base):
835
class SmartTCPClientMedium(SmartClientStreamMedium):
836
"""A client medium using TCP."""
838
def __init__(self, host, port, base):
839
"""Creates a client that will connect on the first use."""
1077
840
SmartClientStreamMedium.__init__(self, base)
841
self._connected = False
1078
844
self._socket = None
1079
self._connected = False
1081
846
def _accept_bytes(self, bytes):
1082
847
"""See SmartClientMedium.accept_bytes."""
1083
848
self._ensure_connection()
1084
849
osutils.send_all(self._socket, bytes, self._report_activity)
1086
def _ensure_connection(self):
1087
"""Connect this medium if not already connected."""
1088
raise NotImplementedError(self._ensure_connection)
1091
"""See SmartClientStreamMedium._flush().
1093
For sockets we do no flushing. For TCP sockets we may want to turn off
1094
TCP_NODELAY and add a means to do a flush, but that can be done in the
1098
def _read_bytes(self, count):
1099
"""See SmartClientMedium.read_bytes."""
1100
if not self._connected:
1101
raise errors.MediumNotConnected(self)
1102
return osutils.read_bytes_from_socket(
1103
self._socket, self._report_activity)
1105
851
def disconnect(self):
1106
852
"""See SmartClientMedium.disconnect()."""
1107
853
if not self._connected:
1130
866
port = int(self._port)
1132
868
sockaddrs = socket.getaddrinfo(self._host, port, socket.AF_UNSPEC,
1133
socket.SOCK_STREAM, 0, 0)
1134
except socket.gaierror as xxx_todo_changeme:
1135
(err_num, err_msg) = xxx_todo_changeme.args
869
socket.SOCK_STREAM, 0, 0)
870
except socket.gaierror, (err_num, err_msg):
1136
871
raise errors.ConnectionError("failed to lookup %s:%d: %s" %
1137
(self._host, port, err_msg))
872
(self._host, port, err_msg))
1138
873
# Initialize err in case there are no addresses returned:
1139
last_err = socket.error("no address found for %s" % self._host)
874
err = socket.error("no address found for %s" % self._host)
1140
875
for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
1142
877
self._socket = socket.socket(family, socktype, proto)
1143
878
self._socket.setsockopt(socket.IPPROTO_TCP,
1144
879
socket.TCP_NODELAY, 1)
1145
880
self._socket.connect(sockaddr)
1146
except socket.error as err:
881
except socket.error, err:
1147
882
if self._socket is not None:
1148
883
self._socket.close()
1149
884
self._socket = None
1153
887
if self._socket is None:
1154
888
# socket errors either have a (string) or (errno, string) as their
1156
if isinstance(last_err.args, str):
1157
err_msg = last_err.args
890
if type(err.args) is str:
1159
err_msg = last_err.args[1]
893
err_msg = err.args[1]
1160
894
raise errors.ConnectionError("failed to connect to %s:%d: %s" %
1161
(self._host, port, err_msg))
1162
self._connected = True
1163
for hook in transport.Transport.hooks["post_connect"]:
1167
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1168
"""A client medium for an already connected socket.
1170
Note that this class will assume it "owns" the socket, so it will close it
1171
when its disconnect method is called.
1174
def __init__(self, base, sock):
1175
SmartClientSocketMedium.__init__(self, base)
1177
self._connected = True
1179
def _ensure_connection(self):
1180
# Already connected, by definition! So nothing to do.
895
(self._host, port, err_msg))
896
self._connected = True
899
"""See SmartClientStreamMedium._flush().
901
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
902
add a means to do a flush, but that can be done in the future.
905
def _read_bytes(self, count):
906
"""See SmartClientMedium.read_bytes."""
907
if not self._connected:
908
raise errors.MediumNotConnected(self)
909
return osutils.read_bytes_from_socket(
910
self._socket, self._report_activity)
1184
913
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
added
removed
bzrlib/smart/medium.py
