28
27
from cStringIO import StringIO
40
from warnings import warn
37
42
from bzrlib import (
43
47
from bzrlib.trace import mutter, warning
44
48
from bzrlib.transport import (
45
AppendBasedFileStream,
48
register_urlparse_netloc_protocol,
53
register_urlparse_netloc_protocol('aftp')
53
from bzrlib.transport.local import LocalURLServer
56
59
class FtpPathError(errors.PathError):
57
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
60
85
class FtpStatResult(object):
62
def __init__(self, f, abspath):
86
def __init__(self, f, relpath):
64
self.st_size = f.size(abspath)
88
self.st_size = f.size(relpath)
65
89
self.st_mode = stat.S_IFREG
66
90
except ftplib.error_perm:
70
94
self.st_mode = stat.S_IFDIR
75
99
_number_of_retries = 2
76
100
_sleep_between_retries = 5
78
# FIXME: there are inconsistencies in the way temporary errors are
79
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
80
# be taken to analyze the implications for write operations (read operations
81
# are safe to retry). Overall even some read operations are never
82
# retried. --vila 20070720 (Bug #127164)
83
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
84
103
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
87
106
"""Set the base path where files will be stored."""
88
if not (base.startswith('ftp://') or base.startswith('aftp://')):
89
raise ValueError(base)
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
98
# Most modern FTP servers support the APPE command. If ours doesn't, we
99
# (re)set this flag accordingly later.
100
self._has_append = True
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
102
137
def _get_FTP(self):
103
138
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
connection_class = ftplib.FTP
114
def _create_connection(self, credentials=None):
115
"""Create a new connection with the provided credentials.
117
:param credentials: The credentials needed to establish the connection.
119
:return: The created connection and its associated credentials.
121
The input credentials are only the password as it may have been
122
entered interactively by the user and may be different from the one
123
provided in base url at transport creation time. The returned
124
credentials are username, password.
126
if credentials is None:
127
user, password = self._user, self._password
129
user, password = credentials
131
auth = config.AuthenticationConfig()
133
user = auth.get_user('ftp', self._host, port=self._port,
134
default=getpass.getuser())
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
139
if self._FTP_instance is not None:
140
return self._FTP_instance
139
connection = self.connection_class()
140
connection.connect(host=self._host, port=self._port)
141
self._login(connection, auth, user, password)
142
connection.set_pasv(not self.is_active)
143
# binary mode is the default
144
connection.voidcmd('TYPE I')
145
except socket.error, e:
146
raise errors.SocketConnectionError(self._host, self._port,
147
msg='Unable to connect to',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
149
147
except ftplib.error_perm, e:
150
raise errors.TransportError(msg="Error setting up connection:"
151
" %s" % str(e), orig_error=e)
152
return connection, (user, password)
154
def _login(self, connection, auth, user, password):
155
# '' is a valid password
156
if user and user != 'anonymous' and password is None:
157
password = auth.get_password('ftp', self._host,
158
user, port=self._port)
159
connection.login(user=user, passwd=password)
161
def _reconnect(self):
162
"""Create a new connection with the previously used credentials"""
163
credentials = self._get_credentials()
164
connection, credentials = self._create_connection(credentials)
165
self._set_connection(connection, credentials)
167
def _translate_ftp_error(self, err, path, extra=None,
168
unknown_exc=FtpPathError):
169
"""Try to translate an ftplib exception to a bzrlib exception.
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
171
154
:param err: The error to translate into a bzr error
172
155
:param path: The path which had problems
186
166
or 'could not open' in s
187
167
or 'no such dir' in s
188
168
or 'could not create file' in s # vsftpd
189
or 'file doesn\'t exist' in s
190
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
191
or 'file/directory not found' in s # filezilla server
192
# Microsoft FTP-Service RNFR reply if file not found
193
or (s.startswith('550 ') and 'unable to rename to' in extra)
195
170
raise errors.NoSuchFile(path, extra=extra)
196
elif ('file exists' in s):
171
if ('file exists' in s):
197
172
raise errors.FileExists(path, extra=extra)
198
elif ('not a directory' in s):
173
if ('not a directory' in s):
199
174
raise errors.PathError(path, extra=extra)
200
elif 'directory not empty' in s:
201
raise errors.DirectoryNotEmpty(path, extra=extra)
203
176
mutter('unable to understand error for path: %s: %s', path, err)
206
179
raise unknown_exc(path, extra=extra)
207
# TODO: jam 20060516 Consider re-raising the error wrapped in
180
# TODO: jam 20060516 Consider re-raising the error wrapped in
208
181
# something like TransportError, but this loses the traceback
209
182
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
210
183
# to handle. Consider doing something like that here.
211
184
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
187
def should_cache(self):
188
"""Return True if the data pulled across should be cached locally.
192
def clone(self, offset=None):
193
"""Return a new FtpTransport with root at self.base + offset.
197
return FtpTransport(self.base, self._FTP_instance)
199
return FtpTransport(self.abspath(offset), self._FTP_instance)
201
def _abspath(self, relpath):
202
assert isinstance(relpath, basestring)
203
relpath = urlutils.unescape(relpath)
204
if relpath.startswith('/'):
207
basepath = self._path.split('/')
208
if len(basepath) > 0 and basepath[-1] == '':
209
basepath = basepath[:-1]
210
for p in relpath.split('/'):
212
if len(basepath) == 0:
213
# In most filesystems, a request for the parent
214
# of root, just returns root.
217
elif p == '.' or p == '':
221
# Possibly, we could use urlparse.urljoin() here, but
222
# I'm concerned about when it chooses to strip the last
223
# portion of the path, and when it doesn't.
225
# XXX: It seems that ftplib does not handle Unicode paths
226
# at the same time, medusa won't handle utf8 paths
227
# So if we .encode(utf8) here, then we get a Server failure.
228
# while if we use str(), we get a UnicodeError, and the test suite
229
# just skips testing UnicodePaths.
230
return str('/'.join(basepath) or '/')
232
def abspath(self, relpath):
233
"""Return the full url to the given relative path.
234
This can be supplied with a string or a list
236
path = self._abspath(relpath)
237
return self._unparse_url(path)
214
239
def has(self, relpath):
215
240
"""Does the target location exist?"""
216
241
# FIXME jam 20060516 We *do* ask about directories in the test suite
342
346
warning("FTP control connection closed. Trying to reopen.")
343
347
time.sleep(_sleep_between_retries)
348
self._FTP_instance = None
345
349
self.put_file(relpath, fp, mode, retries+1)
347
351
def mkdir(self, relpath, mode=None):
348
352
"""Create a directory at the given path."""
349
abspath = self._remote_path(relpath)
353
abspath = self._abspath(relpath)
351
355
mutter("FTP mkd: %s", abspath)
352
356
f = self._get_FTP()
354
self._setmode(relpath, mode)
355
358
except ftplib.error_perm, e:
356
self._translate_ftp_error(e, abspath,
359
self._translate_perm_error(e, abspath,
357
360
unknown_exc=errors.FileExists)
359
def open_write_stream(self, relpath, mode=None):
360
"""See Transport.open_write_stream."""
361
self.put_bytes(relpath, "", mode)
362
result = AppendBasedFileStream(self, relpath)
363
_file_streams[self.abspath(relpath)] = result
366
def recommended_page_size(self):
367
"""See Transport.recommended_page_size().
369
For FTP we suggest a large page size to reduce the overhead
370
introduced by latency.
374
362
def rmdir(self, rel_path):
375
363
"""Delete the directory at rel_path"""
376
abspath = self._remote_path(rel_path)
364
abspath = self._abspath(rel_path)
378
366
mutter("FTP rmd: %s", abspath)
379
367
f = self._get_FTP()
381
369
except ftplib.error_perm, e:
382
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
370
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
384
372
def append_file(self, relpath, f, mode=None):
385
373
"""Append the text in the file-like object into the final
389
abspath = self._remote_path(relpath)
376
abspath = self._abspath(relpath)
390
377
if self.has(relpath):
391
378
ftp = self._get_FTP()
392
379
result = ftp.size(abspath)
397
mutter("FTP appe to %s", abspath)
398
self._try_append(relpath, text, mode)
400
self._fallback_append(relpath, text, mode)
383
mutter("FTP appe to %s", abspath)
384
self._try_append(relpath, f.read(), mode)
404
388
def _try_append(self, relpath, text, mode=None, retries=0):
405
389
"""Try repeatedly to append the given text to the file at relpath.
407
391
This is a recursive function. On errors, it will be called until the
408
392
number of retries is exceeded.
411
abspath = self._remote_path(relpath)
395
abspath = self._abspath(relpath)
412
396
mutter("FTP appe (try %d) to %s", retries, abspath)
413
397
ftp = self._get_FTP()
398
ftp.voidcmd("TYPE I")
414
399
cmd = "APPE %s" % abspath
415
400
conn = ftp.transfercmd(cmd)
416
401
conn.sendall(text)
418
self._setmode(relpath, mode)
404
self._setmode(relpath, mode)
420
406
except ftplib.error_perm, e:
421
# Check whether the command is not supported (reply code 502)
422
if str(e).startswith('502 '):
423
warning("FTP server does not support file appending natively. "
424
"Performance may be severely degraded! (%s)", e)
425
self._has_append = False
426
self._fallback_append(relpath, text, mode)
428
self._translate_ftp_error(e, abspath, extra='error appending',
429
unknown_exc=errors.NoSuchFile)
407
self._translate_perm_error(e, abspath, extra='error appending',
408
unknown_exc=errors.NoSuchFile)
430
409
except ftplib.error_temp, e:
431
410
if retries > _number_of_retries:
432
raise errors.TransportError(
433
"FTP temporary error during APPEND %s. Aborting."
434
% abspath, orig_error=e)
411
raise errors.TransportError("FTP temporary error during APPEND %s." \
412
"Aborting." % abspath, orig_error=e)
436
414
warning("FTP temporary error: %s. Retrying.", str(e))
415
self._FTP_instance = None
438
416
self._try_append(relpath, text, mode, retries+1)
440
def _fallback_append(self, relpath, text, mode = None):
441
remote = self.get(relpath)
442
remote.seek(0, os.SEEK_END)
445
return self.put_file(relpath, remote, mode)
447
418
def _setmode(self, relpath, mode):
448
419
"""Set permissions on a path.
450
421
Only set permissions if the FTP server supports the 'SITE CHMOD'
455
mutter("FTP site chmod: setting permissions to %s on %s",
456
oct(mode), self._remote_path(relpath))
457
ftp = self._get_FTP()
458
cmd = "SITE CHMOD %s %s" % (oct(mode),
459
self._remote_path(relpath))
461
except ftplib.error_perm, e:
462
# Command probably not available on this server
463
warning("FTP Could not set permissions to %s on %s. %s",
464
oct(mode), self._remote_path(relpath), str(e))
425
mutter("FTP site chmod: setting permissions to %s on %s",
426
str(mode), self._abspath(relpath))
427
ftp = self._get_FTP()
428
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
430
except ftplib.error_perm, e:
431
# Command probably not available on this server
432
warning("FTP Could not set permissions to %s on %s. %s",
433
str(mode), self._abspath(relpath), str(e))
466
435
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
467
436
# to copy something to another machine. And you may be able
478
447
def _rename(self, abs_from, abs_to, f):
480
449
f.rename(abs_from, abs_to)
481
except (ftplib.error_temp, ftplib.error_perm), e:
482
self._translate_ftp_error(e, abs_from,
450
except ftplib.error_perm, e:
451
self._translate_perm_error(e, abs_from,
483
452
': unable to rename to %r' % (abs_to))
485
454
def move(self, rel_from, rel_to):
486
455
"""Move the item at rel_from to the location at rel_to"""
487
abs_from = self._remote_path(rel_from)
488
abs_to = self._remote_path(rel_to)
456
abs_from = self._abspath(rel_from)
457
abs_to = self._abspath(rel_to)
490
459
mutter("FTP mv: %s => %s", abs_from, abs_to)
491
460
f = self._get_FTP()
492
461
self._rename_and_overwrite(abs_from, abs_to, f)
493
462
except ftplib.error_perm, e:
494
self._translate_ftp_error(e, abs_from,
495
extra='unable to rename to %r' % (rel_to,),
463
self._translate_perm_error(e, abs_from,
464
extra='unable to rename to %r' % (rel_to,),
496
465
unknown_exc=errors.PathError)
498
467
def _rename_and_overwrite(self, abs_from, abs_to, f):
515
484
mutter("FTP rm: %s", abspath)
516
485
f.delete(abspath)
517
486
except ftplib.error_perm, e:
518
self._translate_ftp_error(e, abspath, 'error deleting',
487
self._translate_perm_error(e, abspath, 'error deleting',
519
488
unknown_exc=errors.NoSuchFile)
521
def external_url(self):
522
"""See bzrlib.transport.Transport.external_url."""
523
# FTP URL's are externally usable.
526
490
def listable(self):
527
491
"""See Transport.listable."""
530
494
def list_dir(self, relpath):
531
495
"""See Transport.list_dir."""
532
basepath = self._remote_path(relpath)
496
basepath = self._abspath(relpath)
533
497
mutter("FTP nlst: %s", basepath)
534
498
f = self._get_FTP()
537
paths = f.nlst(basepath)
538
except ftplib.error_perm, e:
539
self._translate_ftp_error(e, relpath,
540
extra='error with list_dir')
541
except ftplib.error_temp, e:
542
# xs4all's ftp server raises a 450 temp error when listing an
543
# empty directory. Check for that and just return an empty list
544
# in that case. See bug #215522
545
if str(e).lower().startswith('450 no files found'):
546
mutter('FTP Server returned "%s" for nlst.'
547
' Assuming it means empty directory',
552
# Restore binary mode as nlst switch to ascii mode to retrieve file
500
paths = f.nlst(basepath)
501
except ftplib.error_perm, e:
502
self._translate_perm_error(e, relpath, extra='error with list_dir')
556
503
# If FTP.nlst returns paths prefixed by relpath, strip 'em
557
504
if paths and paths[0].startswith(basepath):
558
505
entries = [path[len(basepath)+1:] for path in paths]
609
556
return self.lock_read(relpath)
559
class FtpServer(Server):
560
"""Common code for SFTP server facilities."""
564
self._ftp_server = None
566
self._async_thread = None
571
"""Calculate an ftp url to this server."""
572
return 'ftp://foo:bar@localhost:%d/' % (self._port)
574
# def get_bogus_url(self):
575
# """Return a URL which cannot be connected to."""
576
# return 'ftp://127.0.0.1:1'
578
def log(self, message):
579
"""This is used by medusa.ftp_server to log connections, etc."""
580
self.logs.append(message)
582
def setUp(self, vfs_server=None):
584
raise RuntimeError('Must have medusa to run the FtpServer')
586
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
587
"FtpServer currently assumes local transport, got %s" % vfs_server
589
self._root = os.getcwdu()
590
self._ftp_server = _ftp_server(
591
authorizer=_test_authorizer(root=self._root),
593
port=0, # bind to a random port
595
logger_object=self # Use FtpServer.log() for messages
597
self._port = self._ftp_server.getsockname()[1]
598
# Don't let it loop forever, or handle an infinite number of requests.
599
# In this case it will run for 100s, or 1000 requests
600
self._async_thread = threading.Thread(
601
target=FtpServer._asyncore_loop_ignore_EBADF,
602
kwargs={'timeout':0.1, 'count':1000})
603
self._async_thread.setDaemon(True)
604
self._async_thread.start()
607
"""See bzrlib.transport.Server.tearDown."""
608
# have asyncore release the channel
609
self._ftp_server.del_channel()
611
self._async_thread.join()
614
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
615
"""Ignore EBADF during server shutdown.
617
We close the socket to get the server to shutdown, but this causes
618
select.select() to raise EBADF.
621
asyncore.loop(*args, **kwargs)
622
except select.error, e:
623
if e.args[0] != errno.EBADF:
629
_test_authorizer = None
633
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
636
import medusa.filesys
637
import medusa.ftp_server
643
class test_authorizer(object):
644
"""A custom Authorizer object for running the test suite.
646
The reason we cannot use dummy_authorizer, is because it sets the
647
channel to readonly, which we don't always want to do.
650
def __init__(self, root):
653
def authorize(self, channel, username, password):
654
"""Return (success, reply_string, filesystem)"""
656
return 0, 'No Medusa.', None
658
channel.persona = -1, -1
659
if username == 'anonymous':
660
channel.read_only = 1
662
channel.read_only = 0
664
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
667
class ftp_channel(medusa.ftp_server.ftp_channel):
668
"""Customized ftp channel"""
670
def log(self, message):
671
"""Redirect logging requests."""
672
mutter('_ftp_channel: %s', message)
674
def log_info(self, message, type='info'):
675
"""Redirect logging requests."""
676
mutter('_ftp_channel %s: %s', type, message)
678
def cmd_rnfr(self, line):
679
"""Prepare for renaming a file."""
680
self._renaming = line[1]
681
self.respond('350 Ready for RNTO')
682
# TODO: jam 20060516 in testing, the ftp server seems to
683
# check that the file already exists, or it sends
684
# 550 RNFR command failed
686
def cmd_rnto(self, line):
687
"""Rename a file based on the target given.
689
rnto must be called after calling rnfr.
691
if not self._renaming:
692
self.respond('503 RNFR required first.')
693
pfrom = self.filesystem.translate(self._renaming)
694
self._renaming = None
695
pto = self.filesystem.translate(line[1])
696
if os.path.exists(pto):
697
self.respond('550 RNTO failed: file exists')
700
os.rename(pfrom, pto)
701
except (IOError, OSError), e:
702
# TODO: jam 20060516 return custom responses based on
703
# why the command failed
704
# (bialix 20070418) str(e) on Python 2.5 @ Windows
705
# sometimes don't provide expected error message;
706
# so we obtain such message via os.strerror()
707
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
709
self.respond('550 RNTO failed')
710
# For a test server, we will go ahead and just die
713
self.respond('250 Rename successful.')
715
def cmd_size(self, line):
716
"""Return the size of a file
718
This is overloaded to help the test suite determine if the
719
target is a directory.
722
if not self.filesystem.isfile(filename):
723
if self.filesystem.isdir(filename):
724
self.respond('550 "%s" is a directory' % (filename,))
726
self.respond('550 "%s" is not a file' % (filename,))
728
self.respond('213 %d'
729
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
731
def cmd_mkd(self, line):
732
"""Create a directory.
734
Overloaded because default implementation does not distinguish
735
*why* it cannot make a directory.
738
self.command_not_understood(''.join(line))
742
self.filesystem.mkdir (path)
743
self.respond ('257 MKD command successful.')
744
except (IOError, OSError), e:
745
# (bialix 20070418) str(e) on Python 2.5 @ Windows
746
# sometimes don't provide expected error message;
747
# so we obtain such message via os.strerror()
748
self.respond ('550 error creating directory: %s' %
749
os.strerror(e.errno))
751
self.respond ('550 error creating directory.')
754
class ftp_server(medusa.ftp_server.ftp_server):
755
"""Customize the behavior of the Medusa ftp_server.
757
There are a few warts on the ftp_server, based on how it expects
761
ftp_channel_class = ftp_channel
763
def __init__(self, *args, **kwargs):
764
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
765
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
767
def log(self, message):
768
"""Redirect logging requests."""
769
mutter('_ftp_server: %s', message)
771
def log_info(self, message, type='info'):
772
"""Override the asyncore.log_info so we don't stipple the screen."""
773
mutter('_ftp_server %s: %s', type, message)
775
_test_authorizer = test_authorizer
776
_ftp_channel = ftp_channel
777
_ftp_server = ftp_server
612
782
def get_test_permutations():
613
783
"""Return the permutations to be used in testing."""
614
from bzrlib.tests import ftp_server
615
return [(FtpTransport, ftp_server.FTPTestServer)]
784
if not _setup_medusa():
785
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
788
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
