1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Implementation of Transport over SFTP, using paramiko."""
32
from bzrlib.errors import (FileExists,
33
TransportNotPossible, NoSuchFile, NonRelativePath,
36
from bzrlib.config import config_dir
37
from bzrlib.trace import mutter, warning, error
38
from bzrlib.transport import Transport, register_transport
43
error('The SFTP transport requires paramiko.')
46
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
47
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
49
from paramiko.sftp_attr import SFTPAttributes
50
from paramiko.sftp_file import SFTPFile
51
from paramiko.sftp_client import SFTPClient
53
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
57
def _get_ssh_vendor():
58
"""Find out what version of SSH is on the system."""
60
if _ssh_vendor is not None:
66
p = subprocess.Popen(['ssh', '-V'],
68
stdin=subprocess.PIPE,
69
stdout=subprocess.PIPE,
70
stderr=subprocess.PIPE)
71
returncode = p.returncode
72
stdout, stderr = p.communicate()
76
if 'OpenSSH' in stderr:
77
mutter('ssh implementation is OpenSSH')
78
_ssh_vendor = 'openssh'
79
elif 'SSH Secure Shell' in stderr:
80
mutter('ssh implementation is SSH Corp.')
83
if _ssh_vendor != 'none':
86
# XXX: 20051123 jamesh
87
# A check for putty's plink or lsh would go here.
89
mutter('falling back to paramiko implementation')
94
"""A socket-like object that talks to an ssh subprocess via pipes."""
95
def __init__(self, hostname, port=None, user=None):
96
vendor = _get_ssh_vendor()
97
assert vendor in ['openssh', 'ssh']
98
if vendor == 'openssh':
100
'-oForwardX11=no', '-oForwardAgent=no',
101
'-oClearAllForwardings=yes', '-oProtocol=2',
102
'-oNoHostAuthenticationForLocalhost=yes']
104
args.extend(['-p', str(port)])
106
args.extend(['-l', user])
107
args.extend(['-s', hostname, 'sftp'])
108
elif vendor == 'ssh':
111
args.extend(['-p', str(port)])
113
args.extend(['-l', user])
114
args.extend(['-s', 'sftp', hostname])
116
self.proc = subprocess.Popen(args, close_fds=True,
117
stdin=subprocess.PIPE,
118
stdout=subprocess.PIPE)
120
def send(self, data):
121
return os.write(self.proc.stdin.fileno(), data)
123
def recv(self, count):
124
return os.read(self.proc.stdout.fileno(), count)
127
self.proc.stdin.close()
128
self.proc.stdout.close()
135
# This is a weakref dictionary, so that we can reuse connections
136
# that are still active. Long term, it might be nice to have some
137
# sort of expiration policy, such as disconnect if inactive for
138
# X seconds. But that requires a lot more fanciness.
139
_connected_hosts = weakref.WeakValueDictionary()
141
def load_host_keys():
143
Load system host keys (probably doesn't work on windows) and any
144
"discovered" keys from previous sessions.
146
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
148
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
150
mutter('failed to load system host keys: ' + str(e))
151
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
153
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
155
mutter('failed to load bzr host keys: ' + str(e))
158
def save_host_keys():
160
Save "discovered" host keys in $(config)/ssh_host_keys/.
162
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
163
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
164
if not os.path.isdir(config_dir()):
165
os.mkdir(config_dir())
167
f = open(bzr_hostkey_path, 'w')
168
f.write('# SSH host keys collected by bzr\n')
169
for hostname, keys in BZR_HOSTKEYS.iteritems():
170
for keytype, key in keys.iteritems():
171
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
174
mutter('failed to save bzr host keys: ' + str(e))
178
class SFTPTransportError (TransportError):
181
class SFTPLock(object):
182
"""This fakes a lock in a remote location."""
183
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
184
def __init__(self, path, transport):
185
assert isinstance(transport, SFTPTransport)
187
self.lock_file = None
189
self.lock_path = path + '.write-lock'
190
self.transport = transport
192
self.lock_file = transport._sftp_open_exclusive(self.lock_path)
194
raise LockError('File %r already locked' % (self.path,))
197
"""Should this warn, or actually try to cleanup?"""
199
warn("SFTPLock %r not explicitly unlocked" % (self.path,))
203
if not self.lock_file:
205
self.lock_file.close()
206
self.lock_file = None
208
self.transport.delete(self.lock_path)
209
except (NoSuchFile,):
210
# What specific errors should we catch here?
213
class SFTPTransport (Transport):
215
Transport implementation for SFTP access.
217
_do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
219
def __init__(self, base, clone_from=None):
220
assert base.startswith('sftp://')
221
self._parse_url(base)
222
base = self._unparse_url()
223
super(SFTPTransport, self).__init__(base)
224
if clone_from is None:
227
# use the same ssh connection, etc
228
self._sftp = clone_from._sftp
229
# super saves 'self.base'
231
def should_cache(self):
233
Return True if the data pulled across should be cached locally.
237
def clone(self, offset=None):
239
Return a new SFTPTransport with root at self.base + offset.
240
We share the same SFTP session between such transports, because it's
241
fairly expensive to set them up.
244
return SFTPTransport(self.base, self)
246
return SFTPTransport(self.abspath(offset), self)
248
def abspath(self, relpath):
250
Return the full url to the given relative path.
252
@param relpath: the relative path or path components
253
@type relpath: str or list
255
return self._unparse_url(self._abspath(relpath))
257
def _abspath(self, relpath):
258
"""Return the absolute path segment without the SFTP URL."""
259
# FIXME: share the common code across transports
260
assert isinstance(relpath, basestring)
261
relpath = [urllib.unquote(relpath)]
262
basepath = self._path.split('/')
263
if len(basepath) > 0 and basepath[-1] == '':
264
basepath = basepath[:-1]
268
if len(basepath) == 0:
269
# In most filesystems, a request for the parent
270
# of root, just returns root.
278
path = '/'.join(basepath)
279
# could still be a "relative" path here, but relative on the sftp server
282
def relpath(self, abspath):
283
username, password, host, port, path = self._split_url(abspath)
284
if (username != self._username or host != self._host or
285
port != self._port or not path.startswith(self._path)):
286
raise NonRelativePath('path %r is not under base URL %r'
287
% (abspath, self.base))
289
return path[pl:].lstrip('/')
291
def has(self, relpath):
293
Does the target location exist?
296
self._sftp.stat(self._abspath(relpath))
301
def get(self, relpath, decode=False):
303
Get the file at the given relative path.
305
:param relpath: The relative path to the file
308
path = self._abspath(relpath)
309
f = self._sftp.file(path)
310
if self._do_prefetch and hasattr(f, 'prefetch'):
313
except (IOError, paramiko.SSHException), x:
314
raise NoSuchFile('Error retrieving %s: %s' % (path, str(x)), x)
316
def get_partial(self, relpath, start, length=None):
318
Get just part of a file.
320
:param relpath: Path to the file, relative to base
321
:param start: The starting position to read from
322
:param length: The length to read. A length of None indicates
323
read to the end of the file.
324
:return: A file-like object containing at least the specified bytes.
325
Some implementations may return objects which can be read
326
past this length, but this is not guaranteed.
328
# TODO: implement get_partial_multi to help with knit support
329
f = self.get(relpath)
331
if self._do_prefetch and hasattr(f, 'prefetch'):
335
def put(self, relpath, f):
337
Copy the file-like or string object into the location.
339
:param relpath: Location to put the contents, relative to base.
340
:param f: File-like or string object.
342
final_path = self._abspath(relpath)
343
tmp_relpath = '%s.tmp.%.9f.%d.%d' % (relpath, time.time(),
344
os.getpid(), random.randint(0,0x7FFFFFFF))
345
tmp_abspath = self._abspath(tmp_relpath)
346
fout = self._sftp_open_exclusive(tmp_relpath)
352
self._translate_io_exception(e, relpath)
353
except paramiko.SSHException, x:
354
raise SFTPTransportError('Unable to write file %r' % (relpath,), x)
356
# If we fail, try to clean up the temporary file
357
# before we throw the exception
358
# but don't let another exception mess things up
361
self._sftp.remove(tmp_abspath)
366
# sftp rename doesn't allow overwriting, so play tricks:
367
tmp_safety = 'bzr.tmp.%.9f.%d.%d' % (time.time(), os.getpid(), random.randint(0, 0x7FFFFFFF))
368
tmp_safety = self._abspath(tmp_safety)
370
self._sftp.rename(final_path, tmp_safety)
377
self._sftp.rename(tmp_abspath, final_path)
379
self._translate_io_exception(e, relpath)
380
except paramiko.SSHException, x:
381
raise SFTPTransportError('Unable to rename into file %r' % (path,), x)
387
self._sftp.unlink(tmp_safety)
389
self._sftp.rename(tmp_safety, final_path)
391
def iter_files_recursive(self):
392
"""Walk the relative paths of all files in this transport."""
393
queue = list(self.list_dir('.'))
395
relpath = urllib.quote(queue.pop(0))
396
st = self.stat(relpath)
397
if stat.S_ISDIR(st.st_mode):
398
for i, basename in enumerate(self.list_dir(relpath)):
399
queue.insert(i, relpath+'/'+basename)
403
def mkdir(self, relpath):
404
"""Create a directory at the given path."""
406
path = self._abspath(relpath)
407
self._sftp.mkdir(path)
409
self._translate_io_exception(e, relpath)
410
except (IOError, paramiko.SSHException), x:
411
raise SFTPTransportError('Unable to mkdir %r' % (path,), x)
413
def _translate_io_exception(self, e, relpath):
414
# paramiko seems to generate detailless errors.
415
if (e.errno == errno.ENOENT or
416
e.args == ('No such file or directory',) or
417
e.args == ('No such file',)):
418
raise NoSuchFile(relpath)
419
if (e.args == ('mkdir failed',)):
420
raise FileExists(relpath)
421
# strange but true, for the paramiko server.
422
if (e.args == ('Failure',)):
423
raise FileExists(relpath)
426
def append(self, relpath, f):
428
Append the text in the file-like object into the final
432
path = self._abspath(relpath)
433
fout = self._sftp.file(path, 'ab')
435
except (IOError, paramiko.SSHException), x:
436
raise SFTPTransportError('Unable to append file %r' % (path,), x)
438
def copy(self, rel_from, rel_to):
439
"""Copy the item at rel_from to the location at rel_to"""
440
path_from = self._abspath(rel_from)
441
path_to = self._abspath(rel_to)
442
self._copy_abspaths(path_from, path_to)
444
def _copy_abspaths(self, path_from, path_to):
445
"""Copy files given an absolute path
447
:param path_from: Path on remote server to read
448
:param path_to: Path on remote server to write
451
TODO: Should the destination location be atomically created?
452
This has not been specified
453
TODO: This should use some sort of remote copy, rather than
454
pulling the data locally, and then writing it remotely
457
fin = self._sftp.file(path_from, 'rb')
459
fout = self._sftp.file(path_to, 'wb')
461
fout.set_pipelined(True)
462
self._pump(fin, fout)
467
except (IOError, paramiko.SSHException), x:
468
raise SFTPTransportError('Unable to copy %r to %r' % (path_from, path_to), x)
470
def copy_to(self, relpaths, other, pb=None):
471
"""Copy a set of entries from self into another Transport.
473
:param relpaths: A list/generator of entries to be copied.
475
if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
476
# Both from & to are on the same remote filesystem
477
# We can use a remote copy, instead of pulling locally, and pushing
479
total = self._get_total(relpaths)
481
for path in relpaths:
482
path_from = self._abspath(relpath)
483
path_to = other._abspath(relpath)
484
self._update_pb(pb, 'copy-to', count, total)
485
self._copy_abspaths(path_from, path_to)
489
return super(SFTPTransport, self).copy_to(relpaths, other, pb=pb)
491
# The dummy implementation just does a simple get + put
492
def copy_entry(path):
493
other.put(path, self.get(path))
495
return self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)
497
def move(self, rel_from, rel_to):
498
"""Move the item at rel_from to the location at rel_to"""
499
path_from = self._abspath(rel_from)
500
path_to = self._abspath(rel_to)
502
self._sftp.rename(path_from, path_to)
503
except (IOError, paramiko.SSHException), x:
504
raise SFTPTransportError('Unable to move %r to %r' % (path_from, path_to), x)
506
def delete(self, relpath):
507
"""Delete the item at relpath"""
508
path = self._abspath(relpath)
510
self._sftp.remove(path)
511
except (IOError, paramiko.SSHException), x:
512
raise SFTPTransportError('Unable to delete %r' % (path,), x)
515
"""Return True if this store supports listing."""
518
def list_dir(self, relpath):
520
Return a list of all files at the given location.
522
# does anything actually use this?
523
path = self._abspath(relpath)
525
return self._sftp.listdir(path)
526
except (IOError, paramiko.SSHException), x:
527
raise SFTPTransportError('Unable to list folder %r' % (path,), x)
529
def stat(self, relpath):
530
"""Return the stat information for a file."""
531
path = self._abspath(relpath)
533
return self._sftp.stat(path)
534
except (IOError, paramiko.SSHException), x:
535
raise SFTPTransportError('Unable to stat %r' % (path,), x)
537
def lock_read(self, relpath):
539
Lock the given file for shared (read) access.
540
:return: A lock object, which has an unlock() member function
542
# FIXME: there should be something clever i can do here...
543
class BogusLock(object):
544
def __init__(self, path):
548
return BogusLock(relpath)
550
def lock_write(self, relpath):
552
Lock the given file for exclusive (write) access.
553
WARNING: many transports do not support this, so trying avoid using it
555
:return: A lock object, which has an unlock() member function
557
# This is a little bit bogus, but basically, we create a file
558
# which should not already exist, and if it does, we assume
559
# that there is a lock, and if it doesn't, the we assume
560
# that we have taken the lock.
561
return SFTPLock(relpath, self)
564
def _unparse_url(self, path=None):
567
path = urllib.quote(path)
568
if path.startswith('/'):
569
path = '/%2F' + path[1:]
572
netloc = urllib.quote(self._host)
573
if self._username is not None:
574
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
575
if self._port not in (None, 22):
576
netloc = '%s:%d' % (netloc, self._port)
578
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
580
def _split_url(self, url):
581
if isinstance(url, unicode):
582
url = url.encode('utf-8')
583
(scheme, netloc, path, params,
584
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
585
assert scheme == 'sftp'
586
username = password = host = port = None
588
username, host = netloc.split('@', 1)
590
username, password = username.split(':', 1)
591
password = urllib.unquote(password)
592
username = urllib.unquote(username)
597
host, port = host.rsplit(':', 1)
601
raise SFTPTransportError('%s: invalid port number' % port)
602
host = urllib.unquote(host)
604
path = urllib.unquote(path)
606
# the initial slash should be removed from the path, and treated
607
# as a homedir relative path (the path begins with a double slash
608
# if it is absolute).
609
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
610
if path.startswith('/'):
613
return (username, password, host, port, path)
615
def _parse_url(self, url):
616
(self._username, self._password,
617
self._host, self._port, self._path) = self._split_url(url)
618
if self._port is None:
621
def _sftp_connect(self):
622
"""Connect to the remote sftp server.
623
After this, self._sftp should have a valid connection (or
624
we raise an SFTPTransportError 'could not connect').
626
TODO: Raise a more reasonable ConnectionFailed exception
628
global _connected_hosts
630
idx = (self._host, self._port, self._username)
632
self._sftp = _connected_hosts[idx]
637
vendor = _get_ssh_vendor()
639
sock = SFTPSubprocess(self._host, self._port, self._username)
640
self._sftp = SFTPClient(sock)
642
self._paramiko_connect()
644
_connected_hosts[idx] = self._sftp
646
def _paramiko_connect(self):
647
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
652
t = paramiko.Transport((self._host, self._port))
654
except paramiko.SSHException:
655
raise SFTPTransportError('Unable to reach SSH host %s:%d' % (self._host, self._port))
657
server_key = t.get_remote_server_key()
658
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
659
keytype = server_key.get_name()
660
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
661
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
662
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
663
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
664
our_server_key = BZR_HOSTKEYS[self._host][keytype]
665
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
667
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
668
if not BZR_HOSTKEYS.has_key(self._host):
669
BZR_HOSTKEYS[self._host] = {}
670
BZR_HOSTKEYS[self._host][keytype] = server_key
671
our_server_key = server_key
672
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
674
if server_key != our_server_key:
675
filename1 = os.path.expanduser('~/.ssh/known_hosts')
676
filename2 = os.path.join(config_dir(), 'ssh_host_keys')
677
raise SFTPTransportError('Host keys for %s do not match! %s != %s' % \
678
(self._host, our_server_key_hex, server_key_hex),
679
['Try editing %s or %s' % (filename1, filename2)])
681
self._sftp_auth(t, self._username or getpass.getuser(), self._host)
684
self._sftp = t.open_sftp_client()
685
except paramiko.SSHException:
686
raise BzrError('Unable to find path %s on SFTP server %s' % \
687
(self._path, self._host))
689
def _sftp_auth(self, transport, username, host):
690
agent = paramiko.Agent()
691
for key in agent.get_keys():
692
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
694
transport.auth_publickey(self._username, key)
696
except paramiko.SSHException, e:
699
# okay, try finding id_rsa or id_dss? (posix only)
700
if self._try_pkey_auth(transport, paramiko.RSAKey, 'id_rsa'):
702
if self._try_pkey_auth(transport, paramiko.DSSKey, 'id_dsa'):
707
transport.auth_password(self._username, self._password)
709
except paramiko.SSHException, e:
712
# give up and ask for a password
713
# FIXME: shouldn't be implementing UI this deep into bzrlib
714
enc = sys.stdout.encoding
715
password = getpass.getpass('SSH %s@%s password: ' %
716
(self._username.encode(enc, 'replace'), self._host.encode(enc, 'replace')))
718
transport.auth_password(self._username, password)
719
except paramiko.SSHException:
720
raise SFTPTransportError('Unable to authenticate to SSH host as %s@%s' % \
721
(self._username, self._host))
723
def _try_pkey_auth(self, transport, pkey_class, filename):
724
filename = os.path.expanduser('~/.ssh/' + filename)
726
key = pkey_class.from_private_key_file(filename)
727
transport.auth_publickey(self._username, key)
729
except paramiko.PasswordRequiredException:
730
# FIXME: shouldn't be implementing UI this deep into bzrlib
731
enc = sys.stdout.encoding
732
password = getpass.getpass('SSH %s password: ' %
733
(os.path.basename(filename).encode(enc, 'replace'),))
735
key = pkey_class.from_private_key_file(filename, password)
736
transport.auth_publickey(self._username, key)
738
except paramiko.SSHException:
739
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
740
except paramiko.SSHException:
741
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
746
def _sftp_open_exclusive(self, relpath):
747
"""Open a remote path exclusively.
749
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
750
the file already exists. However it does not expose this
751
at the higher level of SFTPClient.open(), so we have to
754
WARNING: This breaks the SFTPClient abstraction, so it
755
could easily break against an updated version of paramiko.
757
:param relpath: The relative path, where the file should be opened
759
path = self._sftp._adjust_cwd(self._abspath(relpath))
760
attr = SFTPAttributes()
761
mode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
762
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
764
t, msg = self._sftp._request(CMD_OPEN, path, mode, attr)
766
raise SFTPTransportError('Expected an SFTP handle')
767
handle = msg.get_string()
768
return SFTPFile(self._sftp, handle, 'w', -1)
770
self._translate_io_exception(e, relpath)
771
except paramiko.SSHException, x:
772
raise SFTPTransportError('Unable to open file %r' % (path,), x)
added
removed
bzrlib/transport/sftp.py
