28
27
from cStringIO import StringIO
40
from warnings import warn
37
42
from bzrlib import (
43
47
from bzrlib.trace import mutter, warning
44
48
from bzrlib.transport import (
45
AppendBasedFileStream,
48
register_urlparse_netloc_protocol,
53
register_urlparse_netloc_protocol('aftp')
53
from bzrlib.transport.local import LocalURLServer
56
59
class FtpPathError(errors.PathError):
57
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
60
85
class FtpStatResult(object):
62
def __init__(self, f, abspath):
86
def __init__(self, f, relpath):
64
self.st_size = f.size(abspath)
88
self.st_size = f.size(relpath)
65
89
self.st_mode = stat.S_IFREG
66
90
except ftplib.error_perm:
70
94
self.st_mode = stat.S_IFDIR
75
99
_number_of_retries = 2
76
100
_sleep_between_retries = 5
78
# FIXME: there are inconsistencies in the way temporary errors are
79
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
80
# be taken to analyze the implications for write operations (read operations
81
# are safe to retry). Overall even some read operations are never
82
# retried. --vila 20070720 (Bug #127164)
83
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
84
103
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
87
106
"""Set the base path where files will be stored."""
88
if not (base.startswith('ftp://') or base.startswith('aftp://')):
89
raise ValueError(base)
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
98
# Most modern FTP servers support the APPE command. If ours doesn't, we
99
# (re)set this flag accordingly later.
100
self._has_append = True
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
102
137
def _get_FTP(self):
103
138
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
connection_class = ftplib.FTP
114
def _create_connection(self, credentials=None):
115
"""Create a new connection with the provided credentials.
117
:param credentials: The credentials needed to establish the connection.
119
:return: The created connection and its associated credentials.
121
The input credentials are only the password as it may have been
122
entered interactively by the user and may be different from the one
123
provided in base url at transport creation time. The returned
124
credentials are username, password.
126
if credentials is None:
127
user, password = self._user, self._password
129
user, password = credentials
131
auth = config.AuthenticationConfig()
133
user = auth.get_user('ftp', self._host, port=self._port,
134
default=getpass.getuser())
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
139
if self._FTP_instance is not None:
140
return self._FTP_instance
139
connection = self.connection_class()
140
connection.connect(host=self._host, port=self._port)
141
self._login(connection, auth, user, password)
142
connection.set_pasv(not self.is_active)
143
# binary mode is the default
144
connection.voidcmd('TYPE I')
145
except socket.error, e:
146
raise errors.SocketConnectionError(self._host, self._port,
147
msg='Unable to connect to',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
149
147
except ftplib.error_perm, e:
150
raise errors.TransportError(msg="Error setting up connection:"
151
" %s" % str(e), orig_error=e)
152
return connection, (user, password)
154
def _login(self, connection, auth, user, password):
155
# '' is a valid password
156
if user and user != 'anonymous' and password is None:
157
password = auth.get_password('ftp', self._host,
158
user, port=self._port)
159
connection.login(user=user, passwd=password)
161
def _reconnect(self):
162
"""Create a new connection with the previously used credentials"""
163
credentials = self._get_credentials()
164
connection, credentials = self._create_connection(credentials)
165
self._set_connection(connection, credentials)
167
def _translate_ftp_error(self, err, path, extra=None,
168
unknown_exc=FtpPathError):
169
"""Try to translate an ftplib exception to a bzrlib exception.
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
171
154
:param err: The error to translate into a bzr error
172
155
:param path: The path which had problems
187
167
or 'no such dir' in s
188
168
or 'could not create file' in s # vsftpd
189
169
or 'file doesn\'t exist' in s
190
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
191
or 'file/directory not found' in s # filezilla server
192
# Microsoft FTP-Service RNFR reply if file not found
193
or (s.startswith('550 ') and 'unable to rename to' in extra)
195
171
raise errors.NoSuchFile(path, extra=extra)
196
elif ('file exists' in s):
172
if ('file exists' in s):
197
173
raise errors.FileExists(path, extra=extra)
198
elif ('not a directory' in s):
174
if ('not a directory' in s):
199
175
raise errors.PathError(path, extra=extra)
200
elif 'directory not empty' in s:
201
raise errors.DirectoryNotEmpty(path, extra=extra)
203
177
mutter('unable to understand error for path: %s: %s', path, err)
206
180
raise unknown_exc(path, extra=extra)
207
# TODO: jam 20060516 Consider re-raising the error wrapped in
181
# TODO: jam 20060516 Consider re-raising the error wrapped in
208
182
# something like TransportError, but this loses the traceback
209
183
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
210
184
# to handle. Consider doing something like that here.
211
185
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
188
def should_cache(self):
189
"""Return True if the data pulled across should be cached locally.
193
def clone(self, offset=None):
194
"""Return a new FtpTransport with root at self.base + offset.
198
return FtpTransport(self.base, self._FTP_instance)
200
return FtpTransport(self.abspath(offset), self._FTP_instance)
202
def _abspath(self, relpath):
203
assert isinstance(relpath, basestring)
204
relpath = urlutils.unescape(relpath)
205
if relpath.startswith('/'):
208
basepath = self._path.split('/')
209
if len(basepath) > 0 and basepath[-1] == '':
210
basepath = basepath[:-1]
211
for p in relpath.split('/'):
213
if len(basepath) == 0:
214
# In most filesystems, a request for the parent
215
# of root, just returns root.
218
elif p == '.' or p == '':
222
# Possibly, we could use urlparse.urljoin() here, but
223
# I'm concerned about when it chooses to strip the last
224
# portion of the path, and when it doesn't.
226
# XXX: It seems that ftplib does not handle Unicode paths
227
# at the same time, medusa won't handle utf8 paths
228
# So if we .encode(utf8) here, then we get a Server failure.
229
# while if we use str(), we get a UnicodeError, and the test suite
230
# just skips testing UnicodePaths.
231
return str('/'.join(basepath) or '/')
233
def abspath(self, relpath):
234
"""Return the full url to the given relative path.
235
This can be supplied with a string or a list
237
path = self._abspath(relpath)
238
return self._unparse_url(path)
214
240
def has(self, relpath):
215
241
"""Does the target location exist?"""
216
242
# FIXME jam 20060516 We *do* ask about directories in the test suite
342
348
warning("FTP control connection closed. Trying to reopen.")
343
349
time.sleep(_sleep_between_retries)
350
self._FTP_instance = None
345
351
self.put_file(relpath, fp, mode, retries+1)
347
353
def mkdir(self, relpath, mode=None):
348
354
"""Create a directory at the given path."""
349
abspath = self._remote_path(relpath)
355
abspath = self._abspath(relpath)
351
357
mutter("FTP mkd: %s", abspath)
352
358
f = self._get_FTP()
354
self._setmode(relpath, mode)
355
360
except ftplib.error_perm, e:
356
self._translate_ftp_error(e, abspath,
361
self._translate_perm_error(e, abspath,
357
362
unknown_exc=errors.FileExists)
359
def open_write_stream(self, relpath, mode=None):
360
"""See Transport.open_write_stream."""
361
self.put_bytes(relpath, "", mode)
362
result = AppendBasedFileStream(self, relpath)
363
_file_streams[self.abspath(relpath)] = result
366
def recommended_page_size(self):
367
"""See Transport.recommended_page_size().
369
For FTP we suggest a large page size to reduce the overhead
370
introduced by latency.
374
364
def rmdir(self, rel_path):
375
365
"""Delete the directory at rel_path"""
376
abspath = self._remote_path(rel_path)
366
abspath = self._abspath(rel_path)
378
368
mutter("FTP rmd: %s", abspath)
379
369
f = self._get_FTP()
381
371
except ftplib.error_perm, e:
382
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
372
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
384
374
def append_file(self, relpath, f, mode=None):
385
375
"""Append the text in the file-like object into the final
389
abspath = self._remote_path(relpath)
378
abspath = self._abspath(relpath)
390
379
if self.has(relpath):
391
380
ftp = self._get_FTP()
392
381
result = ftp.size(abspath)
397
mutter("FTP appe to %s", abspath)
398
self._try_append(relpath, text, mode)
400
self._fallback_append(relpath, text, mode)
385
mutter("FTP appe to %s", abspath)
386
self._try_append(relpath, f.read(), mode)
404
390
def _try_append(self, relpath, text, mode=None, retries=0):
405
391
"""Try repeatedly to append the given text to the file at relpath.
407
393
This is a recursive function. On errors, it will be called until the
408
394
number of retries is exceeded.
411
abspath = self._remote_path(relpath)
397
abspath = self._abspath(relpath)
412
398
mutter("FTP appe (try %d) to %s", retries, abspath)
413
399
ftp = self._get_FTP()
400
ftp.voidcmd("TYPE I")
414
401
cmd = "APPE %s" % abspath
415
402
conn = ftp.transfercmd(cmd)
416
403
conn.sendall(text)
418
self._setmode(relpath, mode)
406
self._setmode(relpath, mode)
420
408
except ftplib.error_perm, e:
421
# Check whether the command is not supported (reply code 502)
422
if str(e).startswith('502 '):
423
warning("FTP server does not support file appending natively. "
424
"Performance may be severely degraded! (%s)", e)
425
self._has_append = False
426
self._fallback_append(relpath, text, mode)
428
self._translate_ftp_error(e, abspath, extra='error appending',
429
unknown_exc=errors.NoSuchFile)
409
self._translate_perm_error(e, abspath, extra='error appending',
410
unknown_exc=errors.NoSuchFile)
430
411
except ftplib.error_temp, e:
431
412
if retries > _number_of_retries:
432
raise errors.TransportError(
433
"FTP temporary error during APPEND %s. Aborting."
434
% abspath, orig_error=e)
413
raise errors.TransportError("FTP temporary error during APPEND %s." \
414
"Aborting." % abspath, orig_error=e)
436
416
warning("FTP temporary error: %s. Retrying.", str(e))
417
self._FTP_instance = None
438
418
self._try_append(relpath, text, mode, retries+1)
440
def _fallback_append(self, relpath, text, mode = None):
441
remote = self.get(relpath)
442
remote.seek(0, os.SEEK_END)
445
return self.put_file(relpath, remote, mode)
447
420
def _setmode(self, relpath, mode):
448
421
"""Set permissions on a path.
450
423
Only set permissions if the FTP server supports the 'SITE CHMOD'
455
mutter("FTP site chmod: setting permissions to %s on %s",
456
oct(mode), self._remote_path(relpath))
457
ftp = self._get_FTP()
458
cmd = "SITE CHMOD %s %s" % (oct(mode),
459
self._remote_path(relpath))
461
except ftplib.error_perm, e:
462
# Command probably not available on this server
463
warning("FTP Could not set permissions to %s on %s. %s",
464
oct(mode), self._remote_path(relpath), str(e))
427
mutter("FTP site chmod: setting permissions to %s on %s",
428
str(mode), self._abspath(relpath))
429
ftp = self._get_FTP()
430
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
432
except ftplib.error_perm, e:
433
# Command probably not available on this server
434
warning("FTP Could not set permissions to %s on %s. %s",
435
str(mode), self._abspath(relpath), str(e))
466
437
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
467
438
# to copy something to another machine. And you may be able
478
449
def _rename(self, abs_from, abs_to, f):
480
451
f.rename(abs_from, abs_to)
481
except (ftplib.error_temp, ftplib.error_perm), e:
482
self._translate_ftp_error(e, abs_from,
452
except ftplib.error_perm, e:
453
self._translate_perm_error(e, abs_from,
483
454
': unable to rename to %r' % (abs_to))
485
456
def move(self, rel_from, rel_to):
486
457
"""Move the item at rel_from to the location at rel_to"""
487
abs_from = self._remote_path(rel_from)
488
abs_to = self._remote_path(rel_to)
458
abs_from = self._abspath(rel_from)
459
abs_to = self._abspath(rel_to)
490
461
mutter("FTP mv: %s => %s", abs_from, abs_to)
491
462
f = self._get_FTP()
492
463
self._rename_and_overwrite(abs_from, abs_to, f)
493
464
except ftplib.error_perm, e:
494
self._translate_ftp_error(e, abs_from,
495
extra='unable to rename to %r' % (rel_to,),
465
self._translate_perm_error(e, abs_from,
466
extra='unable to rename to %r' % (rel_to,),
496
467
unknown_exc=errors.PathError)
498
469
def _rename_and_overwrite(self, abs_from, abs_to, f):
515
486
mutter("FTP rm: %s", abspath)
516
487
f.delete(abspath)
517
488
except ftplib.error_perm, e:
518
self._translate_ftp_error(e, abspath, 'error deleting',
489
self._translate_perm_error(e, abspath, 'error deleting',
519
490
unknown_exc=errors.NoSuchFile)
521
def external_url(self):
522
"""See bzrlib.transport.Transport.external_url."""
523
# FTP URL's are externally usable.
526
492
def listable(self):
527
493
"""See Transport.listable."""
530
496
def list_dir(self, relpath):
531
497
"""See Transport.list_dir."""
532
basepath = self._remote_path(relpath)
498
basepath = self._abspath(relpath)
533
499
mutter("FTP nlst: %s", basepath)
534
500
f = self._get_FTP()
537
paths = f.nlst(basepath)
538
except ftplib.error_perm, e:
539
self._translate_ftp_error(e, relpath,
540
extra='error with list_dir')
541
except ftplib.error_temp, e:
542
# xs4all's ftp server raises a 450 temp error when listing an
543
# empty directory. Check for that and just return an empty list
544
# in that case. See bug #215522
545
if str(e).lower().startswith('450 no files found'):
546
mutter('FTP Server returned "%s" for nlst.'
547
' Assuming it means empty directory',
552
# Restore binary mode as nlst switch to ascii mode to retrieve file
502
paths = f.nlst(basepath)
503
except ftplib.error_perm, e:
504
self._translate_perm_error(e, relpath, extra='error with list_dir')
556
505
# If FTP.nlst returns paths prefixed by relpath, strip 'em
557
506
if paths and paths[0].startswith(basepath):
558
507
entries = [path[len(basepath)+1:] for path in paths]
609
558
return self.lock_read(relpath)
561
class FtpServer(Server):
562
"""Common code for FTP server facilities."""
566
self._ftp_server = None
568
self._async_thread = None
573
"""Calculate an ftp url to this server."""
574
return 'ftp://foo:bar@localhost:%d/' % (self._port)
576
# def get_bogus_url(self):
577
# """Return a URL which cannot be connected to."""
578
# return 'ftp://127.0.0.1:1'
580
def log(self, message):
581
"""This is used by medusa.ftp_server to log connections, etc."""
582
self.logs.append(message)
584
def setUp(self, vfs_server=None):
586
raise RuntimeError('Must have medusa to run the FtpServer')
588
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
589
"FtpServer currently assumes local transport, got %s" % vfs_server
591
self._root = os.getcwdu()
592
self._ftp_server = _ftp_server(
593
authorizer=_test_authorizer(root=self._root),
595
port=0, # bind to a random port
597
logger_object=self # Use FtpServer.log() for messages
599
self._port = self._ftp_server.getsockname()[1]
600
# Don't let it loop forever, or handle an infinite number of requests.
601
# In this case it will run for 100s, or 1000 requests
602
self._async_thread = threading.Thread(
603
target=FtpServer._asyncore_loop_ignore_EBADF,
604
kwargs={'timeout':0.1, 'count':1000})
605
self._async_thread.setDaemon(True)
606
self._async_thread.start()
609
"""See bzrlib.transport.Server.tearDown."""
610
# have asyncore release the channel
611
self._ftp_server.del_channel()
613
self._async_thread.join()
616
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
617
"""Ignore EBADF during server shutdown.
619
We close the socket to get the server to shutdown, but this causes
620
select.select() to raise EBADF.
623
asyncore.loop(*args, **kwargs)
624
except select.error, e:
625
if e.args[0] != errno.EBADF:
631
_test_authorizer = None
635
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
638
import medusa.filesys
639
import medusa.ftp_server
645
class test_authorizer(object):
646
"""A custom Authorizer object for running the test suite.
648
The reason we cannot use dummy_authorizer, is because it sets the
649
channel to readonly, which we don't always want to do.
652
def __init__(self, root):
655
def authorize(self, channel, username, password):
656
"""Return (success, reply_string, filesystem)"""
658
return 0, 'No Medusa.', None
660
channel.persona = -1, -1
661
if username == 'anonymous':
662
channel.read_only = 1
664
channel.read_only = 0
666
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
669
class ftp_channel(medusa.ftp_server.ftp_channel):
670
"""Customized ftp channel"""
672
def log(self, message):
673
"""Redirect logging requests."""
674
mutter('_ftp_channel: %s', message)
676
def log_info(self, message, type='info'):
677
"""Redirect logging requests."""
678
mutter('_ftp_channel %s: %s', type, message)
680
def cmd_rnfr(self, line):
681
"""Prepare for renaming a file."""
682
self._renaming = line[1]
683
self.respond('350 Ready for RNTO')
684
# TODO: jam 20060516 in testing, the ftp server seems to
685
# check that the file already exists, or it sends
686
# 550 RNFR command failed
688
def cmd_rnto(self, line):
689
"""Rename a file based on the target given.
691
rnto must be called after calling rnfr.
693
if not self._renaming:
694
self.respond('503 RNFR required first.')
695
pfrom = self.filesystem.translate(self._renaming)
696
self._renaming = None
697
pto = self.filesystem.translate(line[1])
698
if os.path.exists(pto):
699
self.respond('550 RNTO failed: file exists')
702
os.rename(pfrom, pto)
703
except (IOError, OSError), e:
704
# TODO: jam 20060516 return custom responses based on
705
# why the command failed
706
# (bialix 20070418) str(e) on Python 2.5 @ Windows
707
# sometimes don't provide expected error message;
708
# so we obtain such message via os.strerror()
709
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
711
self.respond('550 RNTO failed')
712
# For a test server, we will go ahead and just die
715
self.respond('250 Rename successful.')
717
def cmd_size(self, line):
718
"""Return the size of a file
720
This is overloaded to help the test suite determine if the
721
target is a directory.
724
if not self.filesystem.isfile(filename):
725
if self.filesystem.isdir(filename):
726
self.respond('550 "%s" is a directory' % (filename,))
728
self.respond('550 "%s" is not a file' % (filename,))
730
self.respond('213 %d'
731
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
733
def cmd_mkd(self, line):
734
"""Create a directory.
736
Overloaded because default implementation does not distinguish
737
*why* it cannot make a directory.
740
self.command_not_understood(''.join(line))
744
self.filesystem.mkdir (path)
745
self.respond ('257 MKD command successful.')
746
except (IOError, OSError), e:
747
# (bialix 20070418) str(e) on Python 2.5 @ Windows
748
# sometimes don't provide expected error message;
749
# so we obtain such message via os.strerror()
750
self.respond ('550 error creating directory: %s' %
751
os.strerror(e.errno))
753
self.respond ('550 error creating directory.')
756
class ftp_server(medusa.ftp_server.ftp_server):
757
"""Customize the behavior of the Medusa ftp_server.
759
There are a few warts on the ftp_server, based on how it expects
763
ftp_channel_class = ftp_channel
765
def __init__(self, *args, **kwargs):
766
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
767
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
769
def log(self, message):
770
"""Redirect logging requests."""
771
mutter('_ftp_server: %s', message)
773
def log_info(self, message, type='info'):
774
"""Override the asyncore.log_info so we don't stipple the screen."""
775
mutter('_ftp_server %s: %s', type, message)
777
_test_authorizer = test_authorizer
778
_ftp_channel = ftp_channel
779
_ftp_server = ftp_server
612
784
def get_test_permutations():
613
785
"""Return the permutations to be used in testing."""
614
from bzrlib.tests import ftp_server
615
return [(FtpTransport, ftp_server.FTPTestServer)]
786
if not _setup_medusa():
787
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
790
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
