1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
39
from warnings import warn
45
from bzrlib.trace import mutter, warning
46
from bzrlib.transport import (
51
from bzrlib.transport.local import LocalURLServer
57
class FtpPathError(errors.PathError):
58
"""FTP failed for path: %(path)s%(extra)s"""
62
def _find_FTP(hostname, port, username, password, is_active):
63
"""Find an ftplib.FTP instance attached to this triplet."""
64
key = (hostname, port, username, password, is_active)
65
alt_key = (hostname, port, username, '********', is_active)
66
if key not in _FTP_cache:
67
mutter("Constructing FTP instance against %r" % (alt_key,))
70
conn.connect(host=hostname, port=port)
71
if username and username != 'anonymous' and not password:
72
password = bzrlib.ui.ui_factory.get_password(
73
prompt='FTP %(user)s@%(host)s password',
74
user=username, host=hostname)
75
conn.login(user=username, passwd=password)
76
conn.set_pasv(not is_active)
78
_FTP_cache[key] = conn
80
return _FTP_cache[key]
83
class FtpStatResult(object):
84
def __init__(self, f, relpath):
86
self.st_size = f.size(relpath)
87
self.st_mode = stat.S_IFREG
88
except ftplib.error_perm:
92
self.st_mode = stat.S_IFDIR
97
_number_of_retries = 2
98
_sleep_between_retries = 5
100
class FtpTransport(Transport):
101
"""This is the transport agent for ftp:// access."""
103
def __init__(self, base, _provided_instance=None):
104
"""Set the base path where files will be stored."""
105
assert base.startswith('ftp://') or base.startswith('aftp://')
107
self.is_active = base.startswith('aftp://')
109
# urlparse won't handle aftp://
111
if not base.endswith('/'):
113
(self._proto, self._username,
114
self._password, self._host,
115
self._port, self._path) = split_url(base)
116
base = self._unparse_url()
118
super(FtpTransport, self).__init__(base)
119
self._FTP_instance = _provided_instance
121
def _unparse_url(self, path=None):
124
path = urllib.quote(path)
125
netloc = urllib.quote(self._host)
126
if self._username is not None:
127
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
128
if self._port is not None:
129
netloc = '%s:%d' % (netloc, self._port)
133
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
136
"""Return the ftplib.FTP instance for this object."""
137
if self._FTP_instance is not None:
138
return self._FTP_instance
141
self._FTP_instance = _find_FTP(self._host, self._port,
142
self._username, self._password,
144
return self._FTP_instance
145
except ftplib.error_perm, e:
146
raise errors.TransportError(msg="Error setting up connection: %s"
147
% str(e), orig_error=e)
149
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
150
"""Try to translate an ftplib.error_perm exception.
152
:param err: The error to translate into a bzr error
153
:param path: The path which had problems
154
:param extra: Extra information which can be included
155
:param unknown_exc: If None, we will just raise the original exception
156
otherwise we raise unknown_exc(path, extra=extra)
162
extra += ': ' + str(err)
163
if ('no such file' in s
164
or 'could not open' in s
165
or 'no such dir' in s
166
or 'could not create file' in s # vsftpd
168
raise errors.NoSuchFile(path, extra=extra)
169
if ('file exists' in s):
170
raise errors.FileExists(path, extra=extra)
171
if ('not a directory' in s):
172
raise errors.PathError(path, extra=extra)
174
mutter('unable to understand error for path: %s: %s', path, err)
177
raise unknown_exc(path, extra=extra)
178
# TODO: jam 20060516 Consider re-raising the error wrapped in
179
# something like TransportError, but this loses the traceback
180
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
181
# to handle. Consider doing something like that here.
182
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
185
def should_cache(self):
186
"""Return True if the data pulled across should be cached locally.
190
def clone(self, offset=None):
191
"""Return a new FtpTransport with root at self.base + offset.
195
return FtpTransport(self.base, self._FTP_instance)
197
return FtpTransport(self.abspath(offset), self._FTP_instance)
199
def _abspath(self, relpath):
200
assert isinstance(relpath, basestring)
201
relpath = urlutils.unescape(relpath)
202
if relpath.startswith('/'):
205
basepath = self._path.split('/')
206
if len(basepath) > 0 and basepath[-1] == '':
207
basepath = basepath[:-1]
208
for p in relpath.split('/'):
210
if len(basepath) == 0:
211
# In most filesystems, a request for the parent
212
# of root, just returns root.
215
elif p == '.' or p == '':
219
# Possibly, we could use urlparse.urljoin() here, but
220
# I'm concerned about when it chooses to strip the last
221
# portion of the path, and when it doesn't.
223
# XXX: It seems that ftplib does not handle Unicode paths
224
# at the same time, medusa won't handle utf8 paths
225
# So if we .encode(utf8) here, then we get a Server failure.
226
# while if we use str(), we get a UnicodeError, and the test suite
227
# just skips testing UnicodePaths.
228
return str('/'.join(basepath) or '/')
230
def abspath(self, relpath):
231
"""Return the full url to the given relative path.
232
This can be supplied with a string or a list
234
path = self._abspath(relpath)
235
return self._unparse_url(path)
237
def has(self, relpath):
238
"""Does the target location exist?"""
239
# FIXME jam 20060516 We *do* ask about directories in the test suite
240
# We don't seem to in the actual codebase
241
# XXX: I assume we're never asked has(dirname) and thus I use
242
# the FTP size command and assume that if it doesn't raise,
244
abspath = self._abspath(relpath)
247
mutter('FTP has check: %s => %s', relpath, abspath)
249
mutter("FTP has: %s", abspath)
251
except ftplib.error_perm, e:
252
if ('is a directory' in str(e).lower()):
253
mutter("FTP has dir: %s: %s", abspath, e)
255
mutter("FTP has not: %s: %s", abspath, e)
258
def get(self, relpath, decode=False, retries=0):
259
"""Get the file at the given relative path.
261
:param relpath: The relative path to the file
262
:param retries: Number of retries after temporary failures so far
265
We're meant to return a file-like object which bzr will
266
then read from. For now we do this via the magic of StringIO
268
# TODO: decode should be deprecated
270
mutter("FTP get: %s", self._abspath(relpath))
273
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
276
except ftplib.error_perm, e:
277
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
278
except ftplib.error_temp, e:
279
if retries > _number_of_retries:
280
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
281
% self.abspath(relpath),
284
warning("FTP temporary error: %s. Retrying.", str(e))
285
self._FTP_instance = None
286
return self.get(relpath, decode, retries+1)
288
if retries > _number_of_retries:
289
raise errors.TransportError("FTP control connection closed during GET %s."
290
% self.abspath(relpath),
293
warning("FTP control connection closed. Trying to reopen.")
294
time.sleep(_sleep_between_retries)
295
self._FTP_instance = None
296
return self.get(relpath, decode, retries+1)
298
def put_file(self, relpath, fp, mode=None, retries=0):
299
"""Copy the file-like or string object into the location.
301
:param relpath: Location to put the contents, relative to base.
302
:param fp: File-like or string object.
303
:param retries: Number of retries after temporary failures so far
306
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
308
abspath = self._abspath(relpath)
309
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
310
os.getpid(), random.randint(0,0x7FFFFFFF))
311
if getattr(fp, 'read', None) is None:
314
mutter("FTP put: %s", abspath)
317
f.storbinary('STOR '+tmp_abspath, fp)
318
f.rename(tmp_abspath, abspath)
319
except (ftplib.error_temp,EOFError), e:
320
warning("Failure during ftp PUT. Deleting temporary file.")
322
f.delete(tmp_abspath)
324
warning("Failed to delete temporary file on the"
325
" server.\nFile: %s", tmp_abspath)
328
except ftplib.error_perm, e:
329
self._translate_perm_error(e, abspath, extra='could not store')
330
except ftplib.error_temp, e:
331
if retries > _number_of_retries:
332
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
333
% self.abspath(relpath), orig_error=e)
335
warning("FTP temporary error: %s. Retrying.", str(e))
336
self._FTP_instance = None
337
self.put_file(relpath, fp, mode, retries+1)
339
if retries > _number_of_retries:
340
raise errors.TransportError("FTP control connection closed during PUT %s."
341
% self.abspath(relpath), orig_error=e)
343
warning("FTP control connection closed. Trying to reopen.")
344
time.sleep(_sleep_between_retries)
345
self._FTP_instance = None
346
self.put_file(relpath, fp, mode, retries+1)
348
def mkdir(self, relpath, mode=None):
349
"""Create a directory at the given path."""
350
abspath = self._abspath(relpath)
352
mutter("FTP mkd: %s", abspath)
355
except ftplib.error_perm, e:
356
self._translate_perm_error(e, abspath,
357
unknown_exc=errors.FileExists)
359
def rmdir(self, rel_path):
360
"""Delete the directory at rel_path"""
361
abspath = self._abspath(rel_path)
363
mutter("FTP rmd: %s", abspath)
366
except ftplib.error_perm, e:
367
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
369
def append_file(self, relpath, f, mode=None):
370
"""Append the text in the file-like object into the final
373
abspath = self._abspath(relpath)
374
if self.has(relpath):
375
ftp = self._get_FTP()
376
result = ftp.size(abspath)
380
mutter("FTP appe to %s", abspath)
381
self._try_append(relpath, f.read(), mode)
385
def _try_append(self, relpath, text, mode=None, retries=0):
386
"""Try repeatedly to append the given text to the file at relpath.
388
This is a recursive function. On errors, it will be called until the
389
number of retries is exceeded.
392
abspath = self._abspath(relpath)
393
mutter("FTP appe (try %d) to %s", retries, abspath)
394
ftp = self._get_FTP()
395
ftp.voidcmd("TYPE I")
396
cmd = "APPE %s" % abspath
397
conn = ftp.transfercmd(cmd)
401
self._setmode(relpath, mode)
403
except ftplib.error_perm, e:
404
self._translate_perm_error(e, abspath, extra='error appending',
405
unknown_exc=errors.NoSuchFile)
406
except ftplib.error_temp, e:
407
if retries > _number_of_retries:
408
raise errors.TransportError("FTP temporary error during APPEND %s." \
409
"Aborting." % abspath, orig_error=e)
411
warning("FTP temporary error: %s. Retrying.", str(e))
412
self._FTP_instance = None
413
self._try_append(relpath, text, mode, retries+1)
415
def _setmode(self, relpath, mode):
416
"""Set permissions on a path.
418
Only set permissions if the FTP server supports the 'SITE CHMOD'
422
mutter("FTP site chmod: setting permissions to %s on %s",
423
str(mode), self._abspath(relpath))
424
ftp = self._get_FTP()
425
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
427
except ftplib.error_perm, e:
428
# Command probably not available on this server
429
warning("FTP Could not set permissions to %s on %s. %s",
430
str(mode), self._abspath(relpath), str(e))
432
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
433
# to copy something to another machine. And you may be able
434
# to give it its own address as the 'to' location.
435
# So implement a fancier 'copy()'
437
def move(self, rel_from, rel_to):
438
"""Move the item at rel_from to the location at rel_to"""
439
abs_from = self._abspath(rel_from)
440
abs_to = self._abspath(rel_to)
442
mutter("FTP mv: %s => %s", abs_from, abs_to)
444
f.rename(abs_from, abs_to)
445
except ftplib.error_perm, e:
446
self._translate_perm_error(e, abs_from,
447
extra='unable to rename to %r' % (rel_to,),
448
unknown_exc=errors.PathError)
452
def delete(self, relpath):
453
"""Delete the item at relpath"""
454
abspath = self._abspath(relpath)
456
mutter("FTP rm: %s", abspath)
459
except ftplib.error_perm, e:
460
self._translate_perm_error(e, abspath, 'error deleting',
461
unknown_exc=errors.NoSuchFile)
464
"""See Transport.listable."""
467
def list_dir(self, relpath):
468
"""See Transport.list_dir."""
469
basepath = self._abspath(relpath)
470
mutter("FTP nlst: %s", basepath)
473
paths = f.nlst(basepath)
474
except ftplib.error_perm, e:
475
self._translate_perm_error(e, relpath, extra='error with list_dir')
476
# If FTP.nlst returns paths prefixed by relpath, strip 'em
477
if paths and paths[0].startswith(basepath):
478
entries = [path[len(basepath)+1:] for path in paths]
481
# Remove . and .. if present
482
return [urlutils.escape(entry) for entry in entries
483
if entry not in ('.', '..')]
485
def iter_files_recursive(self):
486
"""See Transport.iter_files_recursive.
488
This is cargo-culted from the SFTP transport"""
489
mutter("FTP iter_files_recursive")
490
queue = list(self.list_dir("."))
492
relpath = queue.pop(0)
493
st = self.stat(relpath)
494
if stat.S_ISDIR(st.st_mode):
495
for i, basename in enumerate(self.list_dir(relpath)):
496
queue.insert(i, relpath+"/"+basename)
500
def stat(self, relpath):
501
"""Return the stat information for a file."""
502
abspath = self._abspath(relpath)
504
mutter("FTP stat: %s", abspath)
506
return FtpStatResult(f, abspath)
507
except ftplib.error_perm, e:
508
self._translate_perm_error(e, abspath, extra='error w/ stat')
510
def lock_read(self, relpath):
511
"""Lock the given file for shared (read) access.
512
:return: A lock object, which should be passed to Transport.unlock()
514
# The old RemoteBranch ignore lock for reading, so we will
515
# continue that tradition and return a bogus lock object.
516
class BogusLock(object):
517
def __init__(self, path):
521
return BogusLock(relpath)
523
def lock_write(self, relpath):
524
"""Lock the given file for exclusive (write) access.
525
WARNING: many transports do not support this, so trying avoid using it
527
:return: A lock object, which should be passed to Transport.unlock()
529
return self.lock_read(relpath)
532
class FtpServer(Server):
533
"""Common code for SFTP server facilities."""
537
self._ftp_server = None
539
self._async_thread = None
544
"""Calculate an ftp url to this server."""
545
return 'ftp://foo:bar@localhost:%d/' % (self._port)
547
# def get_bogus_url(self):
548
# """Return a URL which cannot be connected to."""
549
# return 'ftp://127.0.0.1:1'
551
def log(self, message):
552
"""This is used by medusa.ftp_server to log connections, etc."""
553
self.logs.append(message)
555
def setUp(self, vfs_server=None):
557
raise RuntimeError('Must have medusa to run the FtpServer')
559
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
560
"FtpServer currently assumes local transport, got %s" % vfs_server
562
self._root = os.getcwdu()
563
self._ftp_server = _ftp_server(
564
authorizer=_test_authorizer(root=self._root),
566
port=0, # bind to a random port
568
logger_object=self # Use FtpServer.log() for messages
570
self._port = self._ftp_server.getsockname()[1]
571
# Don't let it loop forever, or handle an infinite number of requests.
572
# In this case it will run for 100s, or 1000 requests
573
self._async_thread = threading.Thread(
574
target=FtpServer._asyncore_loop_ignore_EBADF,
575
kwargs={'timeout':0.1, 'count':1000})
576
self._async_thread.setDaemon(True)
577
self._async_thread.start()
580
"""See bzrlib.transport.Server.tearDown."""
581
# have asyncore release the channel
582
self._ftp_server.del_channel()
584
self._async_thread.join()
587
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
588
"""Ignore EBADF during server shutdown.
590
We close the socket to get the server to shutdown, but this causes
591
select.select() to raise EBADF.
594
asyncore.loop(*args, **kwargs)
595
except select.error, e:
596
if e.args[0] != errno.EBADF:
602
_test_authorizer = None
606
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
609
import medusa.filesys
610
import medusa.ftp_server
616
class test_authorizer(object):
617
"""A custom Authorizer object for running the test suite.
619
The reason we cannot use dummy_authorizer, is because it sets the
620
channel to readonly, which we don't always want to do.
623
def __init__(self, root):
626
def authorize(self, channel, username, password):
627
"""Return (success, reply_string, filesystem)"""
629
return 0, 'No Medusa.', None
631
channel.persona = -1, -1
632
if username == 'anonymous':
633
channel.read_only = 1
635
channel.read_only = 0
637
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
640
class ftp_channel(medusa.ftp_server.ftp_channel):
641
"""Customized ftp channel"""
643
def log(self, message):
644
"""Redirect logging requests."""
645
mutter('_ftp_channel: %s', message)
647
def log_info(self, message, type='info'):
648
"""Redirect logging requests."""
649
mutter('_ftp_channel %s: %s', type, message)
651
def cmd_rnfr(self, line):
652
"""Prepare for renaming a file."""
653
self._renaming = line[1]
654
self.respond('350 Ready for RNTO')
655
# TODO: jam 20060516 in testing, the ftp server seems to
656
# check that the file already exists, or it sends
657
# 550 RNFR command failed
659
def cmd_rnto(self, line):
660
"""Rename a file based on the target given.
662
rnto must be called after calling rnfr.
664
if not self._renaming:
665
self.respond('503 RNFR required first.')
666
pfrom = self.filesystem.translate(self._renaming)
667
self._renaming = None
668
pto = self.filesystem.translate(line[1])
670
os.rename(pfrom, pto)
671
except (IOError, OSError), e:
672
# TODO: jam 20060516 return custom responses based on
673
# why the command failed
674
self.respond('550 RNTO failed: %s' % (e,))
676
self.respond('550 RNTO failed')
677
# For a test server, we will go ahead and just die
680
self.respond('250 Rename successful.')
682
def cmd_size(self, line):
683
"""Return the size of a file
685
This is overloaded to help the test suite determine if the
686
target is a directory.
689
if not self.filesystem.isfile(filename):
690
if self.filesystem.isdir(filename):
691
self.respond('550 "%s" is a directory' % (filename,))
693
self.respond('550 "%s" is not a file' % (filename,))
695
self.respond('213 %d'
696
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
698
def cmd_mkd(self, line):
699
"""Create a directory.
701
Overloaded because default implementation does not distinguish
702
*why* it cannot make a directory.
705
self.command_not_understood(''.join(line))
709
self.filesystem.mkdir (path)
710
self.respond ('257 MKD command successful.')
711
except (IOError, OSError), e:
712
self.respond ('550 error creating directory: %s' % (e,))
714
self.respond ('550 error creating directory.')
717
class ftp_server(medusa.ftp_server.ftp_server):
718
"""Customize the behavior of the Medusa ftp_server.
720
There are a few warts on the ftp_server, based on how it expects
724
ftp_channel_class = ftp_channel
726
def __init__(self, *args, **kwargs):
727
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
728
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
730
def log(self, message):
731
"""Redirect logging requests."""
732
mutter('_ftp_server: %s', message)
734
def log_info(self, message, type='info'):
735
"""Override the asyncore.log_info so we don't stipple the screen."""
736
mutter('_ftp_server %s: %s', type, message)
738
_test_authorizer = test_authorizer
739
_ftp_channel = ftp_channel
740
_ftp_server = ftp_server
745
def get_test_permutations():
746
"""Return the permutations to be used in testing."""
747
if not _setup_medusa():
748
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
751
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
