28
27
from cStringIO import StringIO
40
from warnings import warn
37
42
from bzrlib import (
43
47
from bzrlib.trace import mutter, warning
44
48
from bzrlib.transport import (
45
AppendBasedFileStream,
48
register_urlparse_netloc_protocol,
53
register_urlparse_netloc_protocol('aftp')
53
from bzrlib.transport.local import LocalURLServer
56
59
class FtpPathError(errors.PathError):
57
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
60
85
class FtpStatResult(object):
62
def __init__(self, f, abspath):
86
def __init__(self, f, relpath):
64
self.st_size = f.size(abspath)
88
self.st_size = f.size(relpath)
65
89
self.st_mode = stat.S_IFREG
66
90
except ftplib.error_perm:
70
94
self.st_mode = stat.S_IFDIR
75
99
_number_of_retries = 2
76
100
_sleep_between_retries = 5
78
# FIXME: there are inconsistencies in the way temporary errors are
79
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
80
# be taken to analyze the implications for write operations (read operations
81
# are safe to retry). Overall even some read operations are never
82
# retried. --vila 20070720 (Bug #127164)
83
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
84
103
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
87
106
"""Set the base path where files will be stored."""
88
if not (base.startswith('ftp://') or base.startswith('aftp://')):
89
raise ValueError(base)
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
98
# Most modern FTP servers support the APPE command. If ours doesn't, we
99
# (re)set this flag accordingly later.
100
self._has_append = True
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
102
137
def _get_FTP(self):
103
138
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
connection_class = ftplib.FTP
114
def _create_connection(self, credentials=None):
115
"""Create a new connection with the provided credentials.
117
:param credentials: The credentials needed to establish the connection.
119
:return: The created connection and its associated credentials.
121
The input credentials are only the password as it may have been
122
entered interactively by the user and may be different from the one
123
provided in base url at transport creation time. The returned
124
credentials are username, password.
126
if credentials is None:
127
user, password = self._user, self._password
129
user, password = credentials
131
auth = config.AuthenticationConfig()
133
user = auth.get_user('ftp', self._host, port=self._port,
134
default=getpass.getuser())
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
139
if self._FTP_instance is not None:
140
return self._FTP_instance
139
connection = self.connection_class()
140
connection.connect(host=self._host, port=self._port)
141
self._login(connection, auth, user, password)
142
connection.set_pasv(not self.is_active)
143
# binary mode is the default
144
connection.voidcmd('TYPE I')
145
except socket.error, e:
146
raise errors.SocketConnectionError(self._host, self._port,
147
msg='Unable to connect to',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
149
147
except ftplib.error_perm, e:
150
raise errors.TransportError(msg="Error setting up connection:"
151
" %s" % str(e), orig_error=e)
152
return connection, (user, password)
154
def _login(self, connection, auth, user, password):
155
# '' is a valid password
156
if user and user != 'anonymous' and password is None:
157
password = auth.get_password('ftp', self._host,
158
user, port=self._port)
159
connection.login(user=user, passwd=password)
161
def _reconnect(self):
162
"""Create a new connection with the previously used credentials"""
163
credentials = self._get_credentials()
164
connection, credentials = self._create_connection(credentials)
165
self._set_connection(connection, credentials)
167
def _translate_ftp_error(self, err, path, extra=None,
168
unknown_exc=FtpPathError):
169
"""Try to translate an ftplib exception to a bzrlib exception.
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
171
154
:param err: The error to translate into a bzr error
172
155
:param path: The path which had problems
187
167
or 'no such dir' in s
188
168
or 'could not create file' in s # vsftpd
189
169
or 'file doesn\'t exist' in s
190
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
191
or 'file/directory not found' in s # filezilla server
192
# Microsoft FTP-Service RNFR reply if file not found
193
or (s.startswith('550 ') and 'unable to rename to' in extra)
195
171
raise errors.NoSuchFile(path, extra=extra)
196
elif ('file exists' in s):
172
if ('file exists' in s):
197
173
raise errors.FileExists(path, extra=extra)
198
elif ('not a directory' in s):
174
if ('not a directory' in s):
199
175
raise errors.PathError(path, extra=extra)
200
elif 'directory not empty' in s:
201
raise errors.DirectoryNotEmpty(path, extra=extra)
203
177
mutter('unable to understand error for path: %s: %s', path, err)
206
180
raise unknown_exc(path, extra=extra)
207
# TODO: jam 20060516 Consider re-raising the error wrapped in
181
# TODO: jam 20060516 Consider re-raising the error wrapped in
208
182
# something like TransportError, but this loses the traceback
209
183
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
210
184
# to handle. Consider doing something like that here.
211
185
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
188
def should_cache(self):
189
"""Return True if the data pulled across should be cached locally.
193
def clone(self, offset=None):
194
"""Return a new FtpTransport with root at self.base + offset.
198
return FtpTransport(self.base, self._FTP_instance)
200
return FtpTransport(self.abspath(offset), self._FTP_instance)
202
def _abspath(self, relpath):
203
assert isinstance(relpath, basestring)
204
relpath = urlutils.unescape(relpath)
205
if relpath.startswith('/'):
208
basepath = self._path.split('/')
209
if len(basepath) > 0 and basepath[-1] == '':
210
basepath = basepath[:-1]
211
for p in relpath.split('/'):
213
if len(basepath) == 0:
214
# In most filesystems, a request for the parent
215
# of root, just returns root.
218
elif p == '.' or p == '':
222
# Possibly, we could use urlparse.urljoin() here, but
223
# I'm concerned about when it chooses to strip the last
224
# portion of the path, and when it doesn't.
226
# XXX: It seems that ftplib does not handle Unicode paths
227
# at the same time, medusa won't handle utf8 paths
228
# So if we .encode(utf8) here, then we get a Server failure.
229
# while if we use str(), we get a UnicodeError, and the test suite
230
# just skips testing UnicodePaths.
231
return str('/'.join(basepath) or '/')
233
def abspath(self, relpath):
234
"""Return the full url to the given relative path.
235
This can be supplied with a string or a list
237
path = self._abspath(relpath)
238
return self._unparse_url(path)
214
240
def has(self, relpath):
215
241
"""Does the target location exist?"""
216
242
# FIXME jam 20060516 We *do* ask about directories in the test suite
342
348
warning("FTP control connection closed. Trying to reopen.")
343
349
time.sleep(_sleep_between_retries)
350
self._FTP_instance = None
345
351
self.put_file(relpath, fp, mode, retries+1)
347
353
def mkdir(self, relpath, mode=None):
348
354
"""Create a directory at the given path."""
349
abspath = self._remote_path(relpath)
355
abspath = self._abspath(relpath)
351
357
mutter("FTP mkd: %s", abspath)
352
358
f = self._get_FTP()
354
self._setmode(relpath, mode)
355
360
except ftplib.error_perm, e:
356
self._translate_ftp_error(e, abspath,
361
self._translate_perm_error(e, abspath,
357
362
unknown_exc=errors.FileExists)
359
def open_write_stream(self, relpath, mode=None):
360
"""See Transport.open_write_stream."""
361
self.put_bytes(relpath, "", mode)
362
result = AppendBasedFileStream(self, relpath)
363
_file_streams[self.abspath(relpath)] = result
366
def recommended_page_size(self):
367
"""See Transport.recommended_page_size().
369
For FTP we suggest a large page size to reduce the overhead
370
introduced by latency.
374
364
def rmdir(self, rel_path):
375
365
"""Delete the directory at rel_path"""
376
abspath = self._remote_path(rel_path)
366
abspath = self._abspath(rel_path)
378
368
mutter("FTP rmd: %s", abspath)
379
369
f = self._get_FTP()
381
371
except ftplib.error_perm, e:
382
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
372
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
384
374
def append_file(self, relpath, f, mode=None):
385
375
"""Append the text in the file-like object into the final
389
abspath = self._remote_path(relpath)
378
abspath = self._abspath(relpath)
390
379
if self.has(relpath):
391
380
ftp = self._get_FTP()
392
381
result = ftp.size(abspath)
397
mutter("FTP appe to %s", abspath)
398
self._try_append(relpath, text, mode)
400
self._fallback_append(relpath, text, mode)
385
mutter("FTP appe to %s", abspath)
386
self._try_append(relpath, f.read(), mode)
404
390
def _try_append(self, relpath, text, mode=None, retries=0):
405
391
"""Try repeatedly to append the given text to the file at relpath.
407
393
This is a recursive function. On errors, it will be called until the
408
394
number of retries is exceeded.
411
abspath = self._remote_path(relpath)
397
abspath = self._abspath(relpath)
412
398
mutter("FTP appe (try %d) to %s", retries, abspath)
413
399
ftp = self._get_FTP()
400
ftp.voidcmd("TYPE I")
414
401
cmd = "APPE %s" % abspath
415
402
conn = ftp.transfercmd(cmd)
416
403
conn.sendall(text)
418
self._setmode(relpath, mode)
406
self._setmode(relpath, mode)
420
408
except ftplib.error_perm, e:
421
# Check whether the command is not supported (reply code 502)
422
if str(e).startswith('502 '):
423
warning("FTP server does not support file appending natively. "
424
"Performance may be severely degraded! (%s)", e)
425
self._has_append = False
426
self._fallback_append(relpath, text, mode)
428
self._translate_ftp_error(e, abspath, extra='error appending',
429
unknown_exc=errors.NoSuchFile)
409
self._translate_perm_error(e, abspath, extra='error appending',
410
unknown_exc=errors.NoSuchFile)
430
411
except ftplib.error_temp, e:
431
412
if retries > _number_of_retries:
432
raise errors.TransportError(
433
"FTP temporary error during APPEND %s. Aborting."
434
% abspath, orig_error=e)
413
raise errors.TransportError("FTP temporary error during APPEND %s." \
414
"Aborting." % abspath, orig_error=e)
436
416
warning("FTP temporary error: %s. Retrying.", str(e))
417
self._FTP_instance = None
438
418
self._try_append(relpath, text, mode, retries+1)
440
def _fallback_append(self, relpath, text, mode = None):
441
remote = self.get(relpath)
442
remote.seek(0, os.SEEK_END)
445
return self.put_file(relpath, remote, mode)
447
420
def _setmode(self, relpath, mode):
448
421
"""Set permissions on a path.
450
423
Only set permissions if the FTP server supports the 'SITE CHMOD'
455
mutter("FTP site chmod: setting permissions to %s on %s",
456
oct(mode), self._remote_path(relpath))
457
ftp = self._get_FTP()
458
cmd = "SITE CHMOD %s %s" % (oct(mode),
459
self._remote_path(relpath))
461
except ftplib.error_perm, e:
462
# Command probably not available on this server
463
warning("FTP Could not set permissions to %s on %s. %s",
464
oct(mode), self._remote_path(relpath), str(e))
427
mutter("FTP site chmod: setting permissions to %s on %s",
428
str(mode), self._abspath(relpath))
429
ftp = self._get_FTP()
430
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
432
except ftplib.error_perm, e:
433
# Command probably not available on this server
434
warning("FTP Could not set permissions to %s on %s. %s",
435
str(mode), self._abspath(relpath), str(e))
466
437
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
467
438
# to copy something to another machine. And you may be able
478
449
def _rename(self, abs_from, abs_to, f):
480
451
f.rename(abs_from, abs_to)
481
except (ftplib.error_temp, ftplib.error_perm), e:
482
self._translate_ftp_error(e, abs_from,
452
except ftplib.error_perm, e:
453
self._translate_perm_error(e, abs_from,
483
454
': unable to rename to %r' % (abs_to))
485
456
def move(self, rel_from, rel_to):
486
457
"""Move the item at rel_from to the location at rel_to"""
487
abs_from = self._remote_path(rel_from)
488
abs_to = self._remote_path(rel_to)
458
abs_from = self._abspath(rel_from)
459
abs_to = self._abspath(rel_to)
490
461
mutter("FTP mv: %s => %s", abs_from, abs_to)
491
462
f = self._get_FTP()
492
463
self._rename_and_overwrite(abs_from, abs_to, f)
493
464
except ftplib.error_perm, e:
494
self._translate_ftp_error(e, abs_from,
495
extra='unable to rename to %r' % (rel_to,),
465
self._translate_perm_error(e, abs_from,
466
extra='unable to rename to %r' % (rel_to,),
496
467
unknown_exc=errors.PathError)
498
469
def _rename_and_overwrite(self, abs_from, abs_to, f):
530
501
def list_dir(self, relpath):
531
502
"""See Transport.list_dir."""
532
basepath = self._remote_path(relpath)
503
basepath = self._abspath(relpath)
533
504
mutter("FTP nlst: %s", basepath)
534
505
f = self._get_FTP()
537
paths = f.nlst(basepath)
538
except ftplib.error_perm, e:
539
self._translate_ftp_error(e, relpath,
540
extra='error with list_dir')
541
except ftplib.error_temp, e:
542
# xs4all's ftp server raises a 450 temp error when listing an
543
# empty directory. Check for that and just return an empty list
544
# in that case. See bug #215522
545
if str(e).lower().startswith('450 no files found'):
546
mutter('FTP Server returned "%s" for nlst.'
547
' Assuming it means empty directory',
552
# Restore binary mode as nlst switch to ascii mode to retrieve file
507
paths = f.nlst(basepath)
508
except ftplib.error_perm, e:
509
self._translate_perm_error(e, relpath, extra='error with list_dir')
556
510
# If FTP.nlst returns paths prefixed by relpath, strip 'em
557
511
if paths and paths[0].startswith(basepath):
558
512
entries = [path[len(basepath)+1:] for path in paths]
609
563
return self.lock_read(relpath)
566
class FtpServer(Server):
567
"""Common code for FTP server facilities."""
571
self._ftp_server = None
573
self._async_thread = None
578
"""Calculate an ftp url to this server."""
579
return 'ftp://foo:bar@localhost:%d/' % (self._port)
581
# def get_bogus_url(self):
582
# """Return a URL which cannot be connected to."""
583
# return 'ftp://127.0.0.1:1'
585
def log(self, message):
586
"""This is used by medusa.ftp_server to log connections, etc."""
587
self.logs.append(message)
589
def setUp(self, vfs_server=None):
591
raise RuntimeError('Must have medusa to run the FtpServer')
593
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
594
"FtpServer currently assumes local transport, got %s" % vfs_server
596
self._root = os.getcwdu()
597
self._ftp_server = _ftp_server(
598
authorizer=_test_authorizer(root=self._root),
600
port=0, # bind to a random port
602
logger_object=self # Use FtpServer.log() for messages
604
self._port = self._ftp_server.getsockname()[1]
605
# Don't let it loop forever, or handle an infinite number of requests.
606
# In this case it will run for 100s, or 1000 requests
607
self._async_thread = threading.Thread(
608
target=FtpServer._asyncore_loop_ignore_EBADF,
609
kwargs={'timeout':0.1, 'count':1000})
610
self._async_thread.setDaemon(True)
611
self._async_thread.start()
614
"""See bzrlib.transport.Server.tearDown."""
615
# have asyncore release the channel
616
self._ftp_server.del_channel()
618
self._async_thread.join()
621
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
622
"""Ignore EBADF during server shutdown.
624
We close the socket to get the server to shutdown, but this causes
625
select.select() to raise EBADF.
628
asyncore.loop(*args, **kwargs)
629
except select.error, e:
630
if e.args[0] != errno.EBADF:
636
_test_authorizer = None
640
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
643
import medusa.filesys
644
import medusa.ftp_server
650
class test_authorizer(object):
651
"""A custom Authorizer object for running the test suite.
653
The reason we cannot use dummy_authorizer, is because it sets the
654
channel to readonly, which we don't always want to do.
657
def __init__(self, root):
660
def authorize(self, channel, username, password):
661
"""Return (success, reply_string, filesystem)"""
663
return 0, 'No Medusa.', None
665
channel.persona = -1, -1
666
if username == 'anonymous':
667
channel.read_only = 1
669
channel.read_only = 0
671
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
674
class ftp_channel(medusa.ftp_server.ftp_channel):
675
"""Customized ftp channel"""
677
def log(self, message):
678
"""Redirect logging requests."""
679
mutter('_ftp_channel: %s', message)
681
def log_info(self, message, type='info'):
682
"""Redirect logging requests."""
683
mutter('_ftp_channel %s: %s', type, message)
685
def cmd_rnfr(self, line):
686
"""Prepare for renaming a file."""
687
self._renaming = line[1]
688
self.respond('350 Ready for RNTO')
689
# TODO: jam 20060516 in testing, the ftp server seems to
690
# check that the file already exists, or it sends
691
# 550 RNFR command failed
693
def cmd_rnto(self, line):
694
"""Rename a file based on the target given.
696
rnto must be called after calling rnfr.
698
if not self._renaming:
699
self.respond('503 RNFR required first.')
700
pfrom = self.filesystem.translate(self._renaming)
701
self._renaming = None
702
pto = self.filesystem.translate(line[1])
703
if os.path.exists(pto):
704
self.respond('550 RNTO failed: file exists')
707
os.rename(pfrom, pto)
708
except (IOError, OSError), e:
709
# TODO: jam 20060516 return custom responses based on
710
# why the command failed
711
# (bialix 20070418) str(e) on Python 2.5 @ Windows
712
# sometimes don't provide expected error message;
713
# so we obtain such message via os.strerror()
714
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
716
self.respond('550 RNTO failed')
717
# For a test server, we will go ahead and just die
720
self.respond('250 Rename successful.')
722
def cmd_size(self, line):
723
"""Return the size of a file
725
This is overloaded to help the test suite determine if the
726
target is a directory.
729
if not self.filesystem.isfile(filename):
730
if self.filesystem.isdir(filename):
731
self.respond('550 "%s" is a directory' % (filename,))
733
self.respond('550 "%s" is not a file' % (filename,))
735
self.respond('213 %d'
736
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
738
def cmd_mkd(self, line):
739
"""Create a directory.
741
Overloaded because default implementation does not distinguish
742
*why* it cannot make a directory.
745
self.command_not_understood(''.join(line))
749
self.filesystem.mkdir (path)
750
self.respond ('257 MKD command successful.')
751
except (IOError, OSError), e:
752
# (bialix 20070418) str(e) on Python 2.5 @ Windows
753
# sometimes don't provide expected error message;
754
# so we obtain such message via os.strerror()
755
self.respond ('550 error creating directory: %s' %
756
os.strerror(e.errno))
758
self.respond ('550 error creating directory.')
761
class ftp_server(medusa.ftp_server.ftp_server):
762
"""Customize the behavior of the Medusa ftp_server.
764
There are a few warts on the ftp_server, based on how it expects
768
ftp_channel_class = ftp_channel
770
def __init__(self, *args, **kwargs):
771
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
772
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
774
def log(self, message):
775
"""Redirect logging requests."""
776
mutter('_ftp_server: %s', message)
778
def log_info(self, message, type='info'):
779
"""Override the asyncore.log_info so we don't stipple the screen."""
780
mutter('_ftp_server %s: %s', type, message)
782
_test_authorizer = test_authorizer
783
_ftp_channel = ftp_channel
784
_ftp_server = ftp_server
612
789
def get_test_permutations():
613
790
"""Return the permutations to be used in testing."""
614
from bzrlib.tests import ftp_server
615
return [(FtpTransport, ftp_server.FTPTestServer)]
791
if not _setup_medusa():
792
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
795
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
