1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
52
from bzrlib.transport.local import LocalURLServer
58
class FtpPathError(errors.PathError):
59
"""FTP failed for path: %(path)s%(extra)s"""
62
class FtpStatResult(object):
63
def __init__(self, f, relpath):
65
self.st_size = f.size(relpath)
66
self.st_mode = stat.S_IFREG
67
except ftplib.error_perm:
71
self.st_mode = stat.S_IFDIR
76
_number_of_retries = 2
77
_sleep_between_retries = 5
79
# FIXME: there are inconsistencies in the way temporary errors are
80
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
81
# be taken to analyze the implications for write operations (read operations
82
# are safe to retry). Overall even some read operations are never retried.
83
class FtpTransport(ConnectedTransport):
84
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, from_transport=None):
87
"""Set the base path where files will be stored."""
88
assert base.startswith('ftp://') or base.startswith('aftp://')
89
super(FtpTransport, self).__init__(base, from_transport)
90
self._unqualified_scheme = 'ftp'
91
if self._scheme == 'aftp':
94
self.is_active = False
97
"""Return the ftplib.FTP instance for this object."""
98
# Ensures that a connection is established
99
connection = self._get_connection()
100
if connection is None:
101
# First connection ever
102
connection, credentials = self._create_connection()
103
self._set_connection(connection, credentials)
106
def _create_connection(self, credentials=None):
107
"""Create a new connection with the provided credentials.
109
:param credentials: The credentials needed to establish the connection.
111
:return: The created connection and its associated credentials.
113
The credentials are only the password as it may have been entered
114
interactively by the user and may be different from the one provided
115
in base url at transport creation time.
117
if credentials is None:
118
password = self._password
120
password = credentials
122
mutter("Constructing FTP instance against %r" %
123
((self._host, self._port, self._user, '********',
126
connection = ftplib.FTP()
127
connection.connect(host=self._host, port=self._port)
128
if self._user and self._user != 'anonymous' and \
129
password is not None: # '' is a valid password
130
get_password = bzrlib.ui.ui_factory.get_password
131
password = get_password(prompt='FTP %(user)s@%(host)s password',
132
user=self._user, host=self._host)
133
connection.login(user=self._user, passwd=password)
134
connection.set_pasv(not self.is_active)
135
except ftplib.error_perm, e:
136
raise errors.TransportError(msg="Error setting up connection:"
137
" %s" % str(e), orig_error=e)
138
return connection, password
140
def _reconnect(self):
141
"""Create a new connection with the previously used credentials"""
142
credentials = self.get_credentials()
143
connection, credentials = self._create_connection(credentials)
144
self._set_connection(connection, credentials)
146
def _translate_perm_error(self, err, path, extra=None,
147
unknown_exc=FtpPathError):
148
"""Try to translate an ftplib.error_perm exception.
150
:param err: The error to translate into a bzr error
151
:param path: The path which had problems
152
:param extra: Extra information which can be included
153
:param unknown_exc: If None, we will just raise the original exception
154
otherwise we raise unknown_exc(path, extra=extra)
160
extra += ': ' + str(err)
161
if ('no such file' in s
162
or 'could not open' in s
163
or 'no such dir' in s
164
or 'could not create file' in s # vsftpd
166
raise errors.NoSuchFile(path, extra=extra)
167
if ('file exists' in s):
168
raise errors.FileExists(path, extra=extra)
169
if ('not a directory' in s):
170
raise errors.PathError(path, extra=extra)
172
mutter('unable to understand error for path: %s: %s', path, err)
175
raise unknown_exc(path, extra=extra)
176
# TODO: jam 20060516 Consider re-raising the error wrapped in
177
# something like TransportError, but this loses the traceback
178
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
179
# to handle. Consider doing something like that here.
180
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
183
def should_cache(self):
184
"""Return True if the data pulled across should be cached locally.
188
def _remote_path(self, relpath):
189
# XXX: It seems that ftplib does not handle Unicode paths
190
# at the same time, medusa won't handle utf8 paths So if
191
# we .encode(utf8) here (see ConnectedTransport
192
# implementation), then we get a Server failure. while
193
# if we use str(), we get a UnicodeError, and the test
194
# suite just skips testing UnicodePaths.
195
relative = str(urlutils.unescape(relpath))
196
remote_path = self._combine_paths(self._path, relative)
199
def has(self, relpath):
200
"""Does the target location exist?"""
201
# FIXME jam 20060516 We *do* ask about directories in the test suite
202
# We don't seem to in the actual codebase
203
# XXX: I assume we're never asked has(dirname) and thus I use
204
# the FTP size command and assume that if it doesn't raise,
206
abspath = self._remote_path(relpath)
209
mutter('FTP has check: %s => %s', relpath, abspath)
211
mutter("FTP has: %s", abspath)
213
except ftplib.error_perm, e:
214
if ('is a directory' in str(e).lower()):
215
mutter("FTP has dir: %s: %s", abspath, e)
217
mutter("FTP has not: %s: %s", abspath, e)
220
def get(self, relpath, decode=False, retries=0):
221
"""Get the file at the given relative path.
223
:param relpath: The relative path to the file
224
:param retries: Number of retries after temporary failures so far
227
We're meant to return a file-like object which bzr will
228
then read from. For now we do this via the magic of StringIO
230
# TODO: decode should be deprecated
232
mutter("FTP get: %s", self._remote_path(relpath))
235
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
238
except ftplib.error_perm, e:
239
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
240
except ftplib.error_temp, e:
241
if retries > _number_of_retries:
242
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
243
% self.abspath(relpath),
246
warning("FTP temporary error: %s. Retrying.", str(e))
248
return self.get(relpath, decode, retries+1)
250
if retries > _number_of_retries:
251
raise errors.TransportError("FTP control connection closed during GET %s."
252
% self.abspath(relpath),
255
warning("FTP control connection closed. Trying to reopen.")
256
time.sleep(_sleep_between_retries)
258
return self.get(relpath, decode, retries+1)
260
def put_file(self, relpath, fp, mode=None, retries=0):
261
"""Copy the file-like or string object into the location.
263
:param relpath: Location to put the contents, relative to base.
264
:param fp: File-like or string object.
265
:param retries: Number of retries after temporary failures so far
268
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
271
abspath = self._remote_path(relpath)
272
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
273
os.getpid(), random.randint(0,0x7FFFFFFF))
274
if getattr(fp, 'read', None) is None:
277
mutter("FTP put: %s", abspath)
280
f.storbinary('STOR '+tmp_abspath, fp)
281
self._rename_and_overwrite(tmp_abspath, abspath, f)
282
except (ftplib.error_temp,EOFError), e:
283
warning("Failure during ftp PUT. Deleting temporary file.")
285
f.delete(tmp_abspath)
287
warning("Failed to delete temporary file on the"
288
" server.\nFile: %s", tmp_abspath)
291
except ftplib.error_perm, e:
292
self._translate_perm_error(e, abspath, extra='could not store')
293
except ftplib.error_temp, e:
294
if retries > _number_of_retries:
295
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
296
% self.abspath(relpath), orig_error=e)
298
warning("FTP temporary error: %s. Retrying.", str(e))
300
self.put_file(relpath, fp, mode, retries+1)
302
if retries > _number_of_retries:
303
raise errors.TransportError("FTP control connection closed during PUT %s."
304
% self.abspath(relpath), orig_error=e)
306
warning("FTP control connection closed. Trying to reopen.")
307
time.sleep(_sleep_between_retries)
309
self.put_file(relpath, fp, mode, retries+1)
311
def mkdir(self, relpath, mode=None):
312
"""Create a directory at the given path."""
313
abspath = self._remote_path(relpath)
315
mutter("FTP mkd: %s", abspath)
318
except ftplib.error_perm, e:
319
self._translate_perm_error(e, abspath,
320
unknown_exc=errors.FileExists)
322
def rmdir(self, rel_path):
323
"""Delete the directory at rel_path"""
324
abspath = self._remote_path(rel_path)
326
mutter("FTP rmd: %s", abspath)
329
except ftplib.error_perm, e:
330
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
332
def append_file(self, relpath, f, mode=None):
333
"""Append the text in the file-like object into the final
336
abspath = self._remote_path(relpath)
337
if self.has(relpath):
338
ftp = self._get_FTP()
339
result = ftp.size(abspath)
343
mutter("FTP appe to %s", abspath)
344
self._try_append(relpath, f.read(), mode)
348
def _try_append(self, relpath, text, mode=None, retries=0):
349
"""Try repeatedly to append the given text to the file at relpath.
351
This is a recursive function. On errors, it will be called until the
352
number of retries is exceeded.
355
abspath = self._remote_path(relpath)
356
mutter("FTP appe (try %d) to %s", retries, abspath)
357
ftp = self._get_FTP()
358
ftp.voidcmd("TYPE I")
359
cmd = "APPE %s" % abspath
360
conn = ftp.transfercmd(cmd)
364
self._setmode(relpath, mode)
366
except ftplib.error_perm, e:
367
self._translate_perm_error(e, abspath, extra='error appending',
368
unknown_exc=errors.NoSuchFile)
369
except ftplib.error_temp, e:
370
if retries > _number_of_retries:
371
raise errors.TransportError("FTP temporary error during APPEND %s." \
372
"Aborting." % abspath, orig_error=e)
374
warning("FTP temporary error: %s. Retrying.", str(e))
376
self._try_append(relpath, text, mode, retries+1)
378
def _setmode(self, relpath, mode):
379
"""Set permissions on a path.
381
Only set permissions if the FTP server supports the 'SITE CHMOD'
385
mutter("FTP site chmod: setting permissions to %s on %s",
386
str(mode), self._remote_path(relpath))
387
ftp = self._get_FTP()
388
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
390
except ftplib.error_perm, e:
391
# Command probably not available on this server
392
warning("FTP Could not set permissions to %s on %s. %s",
393
str(mode), self._remote_path(relpath), str(e))
395
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
396
# to copy something to another machine. And you may be able
397
# to give it its own address as the 'to' location.
398
# So implement a fancier 'copy()'
400
def rename(self, rel_from, rel_to):
401
abs_from = self._remote_path(rel_from)
402
abs_to = self._remote_path(rel_to)
403
mutter("FTP rename: %s => %s", abs_from, abs_to)
405
return self._rename(abs_from, abs_to, f)
407
def _rename(self, abs_from, abs_to, f):
409
f.rename(abs_from, abs_to)
410
except ftplib.error_perm, e:
411
self._translate_perm_error(e, abs_from,
412
': unable to rename to %r' % (abs_to))
414
def move(self, rel_from, rel_to):
415
"""Move the item at rel_from to the location at rel_to"""
416
abs_from = self._remote_path(rel_from)
417
abs_to = self._remote_path(rel_to)
419
mutter("FTP mv: %s => %s", abs_from, abs_to)
421
self._rename_and_overwrite(abs_from, abs_to, f)
422
except ftplib.error_perm, e:
423
self._translate_perm_error(e, abs_from,
424
extra='unable to rename to %r' % (rel_to,),
425
unknown_exc=errors.PathError)
427
def _rename_and_overwrite(self, abs_from, abs_to, f):
428
"""Do a fancy rename on the remote server.
430
Using the implementation provided by osutils.
432
osutils.fancy_rename(abs_from, abs_to,
433
rename_func=lambda p1, p2: self._rename(p1, p2, f),
434
unlink_func=lambda p: self._delete(p, f))
436
def delete(self, relpath):
437
"""Delete the item at relpath"""
438
abspath = self._remote_path(relpath)
440
self._delete(abspath, f)
442
def _delete(self, abspath, f):
444
mutter("FTP rm: %s", abspath)
446
except ftplib.error_perm, e:
447
self._translate_perm_error(e, abspath, 'error deleting',
448
unknown_exc=errors.NoSuchFile)
451
"""See Transport.listable."""
454
def list_dir(self, relpath):
455
"""See Transport.list_dir."""
456
basepath = self._remote_path(relpath)
457
mutter("FTP nlst: %s", basepath)
460
paths = f.nlst(basepath)
461
except ftplib.error_perm, e:
462
self._translate_perm_error(e, relpath, extra='error with list_dir')
463
# If FTP.nlst returns paths prefixed by relpath, strip 'em
464
if paths and paths[0].startswith(basepath):
465
entries = [path[len(basepath)+1:] for path in paths]
468
# Remove . and .. if present
469
return [urlutils.escape(entry) for entry in entries
470
if entry not in ('.', '..')]
472
def iter_files_recursive(self):
473
"""See Transport.iter_files_recursive.
475
This is cargo-culted from the SFTP transport"""
476
mutter("FTP iter_files_recursive")
477
queue = list(self.list_dir("."))
479
relpath = queue.pop(0)
480
st = self.stat(relpath)
481
if stat.S_ISDIR(st.st_mode):
482
for i, basename in enumerate(self.list_dir(relpath)):
483
queue.insert(i, relpath+"/"+basename)
487
def stat(self, relpath):
488
"""Return the stat information for a file."""
489
abspath = self._remote_path(relpath)
491
mutter("FTP stat: %s", abspath)
493
return FtpStatResult(f, abspath)
494
except ftplib.error_perm, e:
495
self._translate_perm_error(e, abspath, extra='error w/ stat')
497
def lock_read(self, relpath):
498
"""Lock the given file for shared (read) access.
499
:return: A lock object, which should be passed to Transport.unlock()
501
# The old RemoteBranch ignore lock for reading, so we will
502
# continue that tradition and return a bogus lock object.
503
class BogusLock(object):
504
def __init__(self, path):
508
return BogusLock(relpath)
510
def lock_write(self, relpath):
511
"""Lock the given file for exclusive (write) access.
512
WARNING: many transports do not support this, so trying avoid using it
514
:return: A lock object, which should be passed to Transport.unlock()
516
return self.lock_read(relpath)
519
class FtpServer(Server):
520
"""Common code for SFTP server facilities."""
524
self._ftp_server = None
526
self._async_thread = None
531
"""Calculate an ftp url to this server."""
532
return 'ftp://foo:bar@localhost:%d/' % (self._port)
534
# def get_bogus_url(self):
535
# """Return a URL which cannot be connected to."""
536
# return 'ftp://127.0.0.1:1'
538
def log(self, message):
539
"""This is used by medusa.ftp_server to log connections, etc."""
540
self.logs.append(message)
542
def setUp(self, vfs_server=None):
544
raise RuntimeError('Must have medusa to run the FtpServer')
546
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
547
"FtpServer currently assumes local transport, got %s" % vfs_server
549
self._root = os.getcwdu()
550
self._ftp_server = _ftp_server(
551
authorizer=_test_authorizer(root=self._root),
553
port=0, # bind to a random port
555
logger_object=self # Use FtpServer.log() for messages
557
self._port = self._ftp_server.getsockname()[1]
558
# Don't let it loop forever, or handle an infinite number of requests.
559
# In this case it will run for 1000s, or 10000 requests
560
self._async_thread = threading.Thread(
561
target=FtpServer._asyncore_loop_ignore_EBADF,
562
kwargs={'timeout':0.1, 'count':10000})
563
self._async_thread.setDaemon(True)
564
self._async_thread.start()
567
"""See bzrlib.transport.Server.tearDown."""
568
# have asyncore release the channel
569
self._ftp_server.del_channel()
571
self._async_thread.join()
574
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
575
"""Ignore EBADF during server shutdown.
577
We close the socket to get the server to shutdown, but this causes
578
select.select() to raise EBADF.
581
asyncore.loop(*args, **kwargs)
582
# FIXME: If we reach that point, we should raise an exception
583
# explaining that the 'count' parameter in setUp is too low or
584
# testers may wonder why their test just sits there waiting for a
585
# server that is already dead. Note that if the tester waits too
586
# long under pdb the server will also die.
587
except select.error, e:
588
if e.args[0] != errno.EBADF:
594
_test_authorizer = None
598
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
601
import medusa.filesys
602
import medusa.ftp_server
608
class test_authorizer(object):
609
"""A custom Authorizer object for running the test suite.
611
The reason we cannot use dummy_authorizer, is because it sets the
612
channel to readonly, which we don't always want to do.
615
def __init__(self, root):
618
def authorize(self, channel, username, password):
619
"""Return (success, reply_string, filesystem)"""
621
return 0, 'No Medusa.', None
623
channel.persona = -1, -1
624
if username == 'anonymous':
625
channel.read_only = 1
627
channel.read_only = 0
629
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
632
class ftp_channel(medusa.ftp_server.ftp_channel):
633
"""Customized ftp channel"""
635
def log(self, message):
636
"""Redirect logging requests."""
637
mutter('_ftp_channel: %s', message)
639
def log_info(self, message, type='info'):
640
"""Redirect logging requests."""
641
mutter('_ftp_channel %s: %s', type, message)
643
def cmd_rnfr(self, line):
644
"""Prepare for renaming a file."""
645
self._renaming = line[1]
646
self.respond('350 Ready for RNTO')
647
# TODO: jam 20060516 in testing, the ftp server seems to
648
# check that the file already exists, or it sends
649
# 550 RNFR command failed
651
def cmd_rnto(self, line):
652
"""Rename a file based on the target given.
654
rnto must be called after calling rnfr.
656
if not self._renaming:
657
self.respond('503 RNFR required first.')
658
pfrom = self.filesystem.translate(self._renaming)
659
self._renaming = None
660
pto = self.filesystem.translate(line[1])
661
if os.path.exists(pto):
662
self.respond('550 RNTO failed: file exists')
665
os.rename(pfrom, pto)
666
except (IOError, OSError), e:
667
# TODO: jam 20060516 return custom responses based on
668
# why the command failed
669
# (bialix 20070418) str(e) on Python 2.5 @ Windows
670
# sometimes don't provide expected error message;
671
# so we obtain such message via os.strerror()
672
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
674
self.respond('550 RNTO failed')
675
# For a test server, we will go ahead and just die
678
self.respond('250 Rename successful.')
680
def cmd_size(self, line):
681
"""Return the size of a file
683
This is overloaded to help the test suite determine if the
684
target is a directory.
687
if not self.filesystem.isfile(filename):
688
if self.filesystem.isdir(filename):
689
self.respond('550 "%s" is a directory' % (filename,))
691
self.respond('550 "%s" is not a file' % (filename,))
693
self.respond('213 %d'
694
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
696
def cmd_mkd(self, line):
697
"""Create a directory.
699
Overloaded because default implementation does not distinguish
700
*why* it cannot make a directory.
703
self.command_not_understood(''.join(line))
707
self.filesystem.mkdir (path)
708
self.respond ('257 MKD command successful.')
709
except (IOError, OSError), e:
710
# (bialix 20070418) str(e) on Python 2.5 @ Windows
711
# sometimes don't provide expected error message;
712
# so we obtain such message via os.strerror()
713
self.respond ('550 error creating directory: %s' %
714
os.strerror(e.errno))
716
self.respond ('550 error creating directory.')
719
class ftp_server(medusa.ftp_server.ftp_server):
720
"""Customize the behavior of the Medusa ftp_server.
722
There are a few warts on the ftp_server, based on how it expects
726
ftp_channel_class = ftp_channel
728
def __init__(self, *args, **kwargs):
729
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
730
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
732
def log(self, message):
733
"""Redirect logging requests."""
734
mutter('_ftp_server: %s', message)
736
def log_info(self, message, type='info'):
737
"""Override the asyncore.log_info so we don't stipple the screen."""
738
mutter('_ftp_server %s: %s', type, message)
740
_test_authorizer = test_authorizer
741
_ftp_channel = ftp_channel
742
_ftp_server = ftp_server
747
def get_test_permutations():
748
"""Return the permutations to be used in testing."""
749
if not _setup_medusa():
750
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
753
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
