44
from bzrlib.errors import (FileExists,
51
from bzrlib.osutils import fancy_rename
52
from bzrlib.trace import mutter, warning
53
from bzrlib.transport import (
42
from ..errors import (FileExists,
49
from ..osutils import fancy_rename
50
from ..trace import mutter, warning
51
from ..transport import (
66
64
# /var/lib/python-support/python2.5/paramiko/message.py:226: DeprecationWarning: integer argument expected, got float
67
65
# self.packet.write(struct.pack('>I', n))
68
66
warnings.filterwarnings('ignore',
69
'integer argument expected, got float',
70
category=DeprecationWarning,
71
module='paramiko.message')
67
'integer argument expected, got float',
68
category=DeprecationWarning,
69
module='paramiko.message')
75
except ImportError, e:
73
except ImportError as e:
76
74
raise ParamikoNotPresent(e)
78
76
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
79
77
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
80
SFTP_OK, CMD_HANDLE, CMD_OPEN)
81
79
from paramiko.sftp_attr import SFTPAttributes
82
80
from paramiko.sftp_file import SFTPFile
85
_paramiko_version = getattr(paramiko, '__version_info__', (0, 0, 0))
86
# don't use prefetch unless paramiko version >= 1.5.5 (there were bugs earlier)
87
_default_do_prefetch = (_paramiko_version >= (1, 5, 5))
83
# GZ 2017-05-25: Some dark hackery to monkeypatch out issues with paramiko's
84
# Python 3 compatibility code. Replace broken b() and asbytes() code.
86
from paramiko.py3compat import b as _bad
87
from paramiko.common import asbytes as _bad_asbytes
91
def _b_for_broken_paramiko(s, encoding='utf8'):
92
"""Hacked b() that does not raise TypeError."""
93
# https://github.com/paramiko/paramiko/issues/967
94
if not isinstance(s, bytes):
95
encode = getattr(s, 'encode', None)
96
if encode is not None:
97
return encode(encoding)
98
# Would like to pass buffer objects along, but have to realise.
99
tobytes = getattr(s, 'tobytes', None)
100
if tobytes is not None:
104
def _asbytes_for_broken_paramiko(s):
105
"""Hacked asbytes() that does not raise Exception."""
106
# https://github.com/paramiko/paramiko/issues/968
107
if not isinstance(s, bytes):
108
encode = getattr(s, 'encode', None)
109
if encode is not None:
110
return encode('utf8')
111
asbytes = getattr(s, 'asbytes', None)
112
if asbytes is not None:
116
_bad.__code__ = _b_for_broken_paramiko.__code__
117
_bad_asbytes.__code__ = _asbytes_for_broken_paramiko.__code__
90
120
class SFTPLock(object):
158
188
# as possible, so we don't issues requests <32kB
159
189
sorted_offsets = sorted(self.original_offsets)
160
190
coalesced = list(ConnectedTransport._coalesce_offsets(sorted_offsets,
161
limit=0, fudge_factor=0))
191
limit=0, fudge_factor=0))
163
193
for c_offset in coalesced:
164
194
start = c_offset.start
172
202
start += next_size
173
203
if 'sftp' in debug.debug_flags:
174
204
mutter('SFTP.readv(%s) %s offsets => %s coalesced => %s requests',
175
self.relpath, len(sorted_offsets), len(coalesced),
205
self.relpath, len(sorted_offsets), len(coalesced),
179
209
def request_and_yield_offsets(self, fp):
186
216
requests = self._get_requests()
187
217
offset_iter = iter(self.original_offsets)
188
cur_offset, cur_size = offset_iter.next()
218
cur_offset, cur_size = next(offset_iter)
189
219
# paramiko .readv() yields strings that are in the order of the requests
190
220
# So we track the current request to know where the next data is
191
221
# being returned from.
203
233
data_stream = itertools.chain(fp.readv(requests),
204
234
itertools.repeat(None))
205
for (start, length), data in itertools.izip(requests, data_stream):
235
for (start, length), data in zip(requests, data_stream):
207
237
if cur_coalesced is not None:
208
238
raise errors.ShortReadvError(self.relpath,
209
start, length, len(data))
239
start, length, len(data))
210
240
if len(data) != length:
211
241
raise errors.ShortReadvError(self.relpath,
212
start, length, len(data))
242
start, length, len(data))
213
243
self._report_activity(length, 'read')
214
244
if last_end is None:
215
245
# This is the first request, just buffer it
259
289
input_start += cur_size
260
290
# Yield the requested data
261
291
yield cur_offset, cur_data
262
cur_offset, cur_size = offset_iter.next()
293
cur_offset, cur_size = next(offset_iter)
294
except StopIteration:
263
296
# at this point, we've consumed as much of buffered as we can,
264
297
# so break off the portion that we consumed
265
298
if buffered_offset == len(buffered_data):
273
306
# now that the data stream is done, close the handle
276
buffered = ''.join(buffered_data)
309
buffered = b''.join(buffered_data)
277
310
del buffered_data[:]
278
311
data_chunks.append((input_start, buffered))
280
313
if 'sftp' in debug.debug_flags:
281
314
mutter('SFTP readv left with %d out-of-order bytes',
282
sum(map(lambda x: len(x[1]), data_chunks)))
315
sum(len(x[1]) for x in data_chunks))
283
316
# We've processed all the readv data, at this point, anything we
284
317
# couldn't process is in data_chunks. This doesn't happen often, so
285
318
# this code path isn't optimized
306
339
if len(data) != cur_size:
307
340
raise AssertionError('We must have miscalulated.'
308
' We expected %d bytes, but only found %d'
309
% (cur_size, len(data)))
341
' We expected %d bytes, but only found %d'
342
% (cur_size, len(data)))
310
343
yield cur_offset, data
311
cur_offset, cur_size = offset_iter.next()
345
cur_offset, cur_size = next(offset_iter)
346
except StopIteration:
314
350
class SFTPTransport(ConnectedTransport):
315
351
"""Transport implementation for SFTP access."""
317
_do_prefetch = _default_do_prefetch
318
353
# TODO: jam 20060717 Conceivably these could be configurable, either
319
354
# by auto-tuning at run-time, or by a configuration (per host??)
320
355
# but the performance curve is pretty flat, so just going with
372
407
auth = config.AuthenticationConfig()
373
408
user = auth.get_user('ssh', self._parsed_url.host,
374
self._parsed_url.port)
409
self._parsed_url.port)
375
410
connection = vendor.connect_sftp(self._parsed_url.user, password,
376
self._parsed_url.host, self._parsed_url.port)
411
self._parsed_url.host, self._parsed_url.port)
377
412
return connection, (user, password)
379
414
def disconnect(self):
411
446
path = self._remote_path(relpath)
412
447
f = self._get_sftp().file(path, mode='rb')
413
if self._do_prefetch and (getattr(f, 'prefetch', None) is not None):
448
size = f.stat().st_size
449
if getattr(f, 'prefetch', None) is not None:
416
except (IOError, paramiko.SSHException), e:
452
except (IOError, paramiko.SSHException) as e:
417
453
self._translate_io_exception(e, path, ': error retrieving',
418
failure_exc=errors.ReadError)
454
failure_exc=errors.ReadError)
420
456
def get_bytes(self, relpath):
421
457
# reimplement this here so that we can report how many bytes came back
422
f = self.get(relpath)
458
with self.get(relpath) as f:
425
460
self._report_activity(len(bytes), 'read')
430
463
def _readv(self, relpath, offsets):
431
464
"""See Transport.readv()"""
444
477
if 'sftp' in debug.debug_flags:
445
478
mutter('seek and read %s offsets', len(offsets))
446
479
return self._seek_and_read(fp, offsets, relpath)
447
except (IOError, paramiko.SSHException), e:
480
except (IOError, paramiko.SSHException) as e:
448
481
self._translate_io_exception(e, path, ': error retrieving')
450
483
def recommended_page_size(self):
479
512
def _put(self, abspath, f, mode=None):
480
513
"""Helper function so both put() and copy_abspaths can reuse the code"""
481
514
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
482
os.getpid(), random.randint(0,0x7FFFFFFF))
515
os.getpid(), random.randint(0, 0x7FFFFFFF))
483
516
fout = self._sftp_open_exclusive(tmp_abspath, mode=mode)
487
520
fout.set_pipelined(True)
488
521
length = self._pump(f, fout)
489
except (IOError, paramiko.SSHException), e:
522
except (IOError, paramiko.SSHException) as e:
490
523
self._translate_io_exception(e, tmp_abspath)
491
524
# XXX: This doesn't truly help like we would like it to.
492
525
# The problem is that openssh strips sticky bits. So while we
596
629
def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None,
597
630
create_parent_dir=False,
599
if not isinstance(raw_bytes, str):
632
if not isinstance(raw_bytes, bytes):
601
634
'raw_bytes must be a plain string, not %s' % type(raw_bytes))
615
648
st = self.stat(relpath)
616
649
if stat.S_ISDIR(st.st_mode):
617
650
for i, basename in enumerate(self.list_dir(relpath)):
618
queue.insert(i, relpath+'/'+basename)
651
queue.insert(i, relpath + '/' + basename)
622
655
def _mkdir(self, abspath, mode=None):
626
659
local_mode = mode
635
668
# the sgid bit is set, report a warning to the user
636
669
# with the umask fix.
637
670
stat = self._get_sftp().lstat(abspath)
638
mode = mode & 0777 # can't set special bits anyway
639
if mode != stat.st_mode & 0777:
640
if stat.st_mode & 06000:
671
mode = mode & 0o777 # can't set special bits anyway
672
if mode != stat.st_mode & 0o777:
673
if stat.st_mode & 0o6000:
641
674
warning('About to chmod %s over sftp, which will result'
642
675
' in its suid or sgid bits being cleared. If'
643
676
' you want to preserve those bits, change your '
644
677
' environment on the server to use umask 0%03o.'
645
% (abspath, 0777 - mode))
678
% (abspath, 0o777 - mode))
646
679
self._get_sftp().chmod(abspath, mode=mode)
647
except (paramiko.SSHException, IOError), e:
680
except (paramiko.SSHException, IOError) as e:
648
681
self._translate_io_exception(e, abspath, ': unable to mkdir',
649
failure_exc=FileExists)
682
failure_exc=FileExists)
651
684
def mkdir(self, relpath, mode=None):
652
685
"""Create a directory at the given path."""
659
692
# api more than once per write_group at the moment so
660
693
# it is a tolerable overhead. Better would be to truncate
661
694
# the file after opening. RBC 20070805
662
self.put_bytes_non_atomic(relpath, "", mode)
695
self.put_bytes_non_atomic(relpath, b"", mode)
663
696
abspath = self._remote_path(relpath)
664
697
# TODO: jam 20060816 paramiko doesn't publicly expose a way to
665
698
# set the file mode at create time. If it does, use it.
669
702
handle = self._get_sftp().file(abspath, mode='wb')
670
703
handle.set_pipelined(True)
671
except (paramiko.SSHException, IOError), e:
704
except (paramiko.SSHException, IOError) as e:
672
705
self._translate_io_exception(e, abspath,
673
706
': unable to open')
674
707
_file_streams[self.abspath(relpath)] = handle
692
725
self._translate_error(e, path, raise_generic=False)
693
726
if getattr(e, 'args', None) is not None:
694
727
if (e.args == ('No such file or directory',) or
695
e.args == ('No such file',)):
728
e.args == ('No such file',)):
696
729
raise NoSuchFile(path, str(e) + more_info)
697
730
if (e.args == ('mkdir failed',) or
698
e.args[0].startswith('syserr: File exists')):
731
e.args[0].startswith('syserr: File exists')):
699
732
raise FileExists(path, str(e) + more_info)
700
733
# strange but true, for the paramiko server.
701
734
if (e.args == ('Failure',)):
704
737
# '/srv/bazaar.launchpad.net/blah...: '
705
738
# [Errno 39] Directory not empty',)
706
739
if (e.args[0].startswith('Directory not empty: ')
707
or getattr(e, 'errno', None) == errno.ENOTEMPTY):
740
or getattr(e, 'errno', None) == errno.ENOTEMPTY):
708
741
raise errors.DirectoryNotEmpty(path, str(e))
709
742
if e.args == ('Operation unsupported',):
710
743
raise errors.TransportNotPossible()
726
759
result = fout.tell()
727
760
self._pump(f, fout)
729
except (IOError, paramiko.SSHException), e:
762
except (IOError, paramiko.SSHException) as e:
730
763
self._translate_io_exception(e, relpath, ': unable to append')
732
765
def rename(self, rel_from, rel_to):
733
766
"""Rename without special overwriting"""
735
768
self._get_sftp().rename(self._remote_path(rel_from),
736
self._remote_path(rel_to))
737
except (IOError, paramiko.SSHException), e:
769
self._remote_path(rel_to))
770
except (IOError, paramiko.SSHException) as e:
738
771
self._translate_io_exception(e, rel_from,
739
': unable to rename to %r' % (rel_to))
772
': unable to rename to %r' % (rel_to))
741
774
def _rename_and_overwrite(self, abs_from, abs_to):
742
775
"""Do a fancy rename on the remote server.
748
781
fancy_rename(abs_from, abs_to,
749
782
rename_func=sftp.rename,
750
783
unlink_func=sftp.remove)
751
except (IOError, paramiko.SSHException), e:
784
except (IOError, paramiko.SSHException) as e:
752
785
self._translate_io_exception(e, abs_from,
753
786
': unable to rename to %r' % (abs_to))
763
796
path = self._remote_path(relpath)
765
798
self._get_sftp().remove(path)
766
except (IOError, paramiko.SSHException), e:
799
except (IOError, paramiko.SSHException) as e:
767
800
self._translate_io_exception(e, path, ': unable to delete')
769
802
def external_url(self):
770
"""See bzrlib.transport.Transport.external_url."""
803
"""See breezy.transport.Transport.external_url."""
771
804
# the external path for SFTP is the base
788
821
entries = self._get_sftp().listdir(path)
789
822
self._report_activity(sum(map(len, entries)), 'read')
790
except (IOError, paramiko.SSHException), e:
823
except (IOError, paramiko.SSHException) as e:
791
824
self._translate_io_exception(e, path, ': failed to list_dir')
792
825
return [urlutils.escape(entry) for entry in entries]
796
829
path = self._remote_path(relpath)
798
831
return self._get_sftp().rmdir(path)
799
except (IOError, paramiko.SSHException), e:
832
except (IOError, paramiko.SSHException) as e:
800
833
self._translate_io_exception(e, path, ': failed to rmdir')
802
835
def stat(self, relpath):
804
837
path = self._remote_path(relpath)
806
839
return self._get_sftp().lstat(path)
807
except (IOError, paramiko.SSHException), e:
840
except (IOError, paramiko.SSHException) as e:
808
841
self._translate_io_exception(e, path, ': unable to stat')
810
843
def readlink(self, relpath):
811
844
"""See Transport.readlink."""
812
845
path = self._remote_path(relpath)
814
return self._get_sftp().readlink(path)
815
except (IOError, paramiko.SSHException), e:
847
return self._get_sftp().readlink(self._remote_path(path))
848
except (IOError, paramiko.SSHException) as e:
816
849
self._translate_io_exception(e, path, ': unable to readlink')
818
851
def symlink(self, source, link_name):
819
852
"""See Transport.symlink."""
821
854
conn = self._get_sftp()
822
sftp_retval = conn.symlink(source, link_name)
823
if SFTP_OK != sftp_retval:
824
raise TransportError(
825
'%r: unable to create symlink to %r' % (link_name, source),
828
except (IOError, paramiko.SSHException), e:
855
sftp_retval = conn.symlink(source, self._remote_path(link_name))
856
except (IOError, paramiko.SSHException) as e:
829
857
self._translate_io_exception(e, link_name,
830
858
': unable to create symlink to %r' % (source))
879
914
if mode is not None:
880
915
attr.st_mode = mode
881
916
omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
882
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
917
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
884
919
t, msg = self._get_sftp()._request(CMD_OPEN, path, omode, attr)
885
920
if t != CMD_HANDLE:
886
921
raise TransportError('Expected an SFTP handle')
887
922
handle = msg.get_string()
888
923
return SFTPFile(self._get_sftp(), handle, 'wb', -1)
889
except (paramiko.SSHException, IOError), e:
924
except (paramiko.SSHException, IOError) as e:
890
925
self._translate_io_exception(e, abspath, ': unable to open',
891
failure_exc=FileExists)
926
failure_exc=FileExists)
893
928
def _can_roundtrip_unix_modebits(self):
894
929
if sys.platform == 'win32':
901
936
def get_test_permutations():
902
937
"""Return the permutations to be used in testing."""
903
from bzrlib.tests import stub_sftp
938
from ..tests import stub_sftp
904
939
return [(SFTPTransport, stub_sftp.SFTPAbsoluteServer),
905
940
(SFTPTransport, stub_sftp.SFTPHomeDirServer),
906
941
(SFTPTransport, stub_sftp.SFTPSiblingAbsoluteServer),