1
# Copyright (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
38
from warnings import warn
44
from bzrlib.trace import mutter, warning
45
from bzrlib.transport import (
55
class FtpPathError(errors.PathError):
56
"""FTP failed for path: %(path)s%(extra)s"""
60
def _find_FTP(hostname, port, username, password, is_active):
61
"""Find an ftplib.FTP instance attached to this triplet."""
62
key = (hostname, port, username, password, is_active)
63
alt_key = (hostname, port, username, '********', is_active)
64
if key not in _FTP_cache:
65
mutter("Constructing FTP instance against %r" % (alt_key,))
68
conn.connect(host=hostname, port=port)
69
if username and username != 'anonymous' and not password:
70
password = bzrlib.ui.ui_factory.get_password(
71
prompt='FTP %(user)s@%(host)s password',
72
user=username, host=hostname)
73
conn.login(user=username, passwd=password)
74
conn.set_pasv(not is_active)
76
_FTP_cache[key] = conn
78
return _FTP_cache[key]
81
class FtpStatResult(object):
82
def __init__(self, f, relpath):
84
self.st_size = f.size(relpath)
85
self.st_mode = stat.S_IFREG
86
except ftplib.error_perm:
90
self.st_mode = stat.S_IFDIR
95
_number_of_retries = 2
96
_sleep_between_retries = 5
98
class FtpTransport(Transport):
99
"""This is the transport agent for ftp:// access."""
101
def __init__(self, base, _provided_instance=None):
102
"""Set the base path where files will be stored."""
103
assert base.startswith('ftp://') or base.startswith('aftp://')
105
self.is_active = base.startswith('aftp://')
107
# urlparse won't handle aftp://
109
if not base.endswith('/'):
111
(self._proto, self._username,
112
self._password, self._host,
113
self._port, self._path) = split_url(base)
114
base = self._unparse_url()
116
super(FtpTransport, self).__init__(base)
117
self._FTP_instance = _provided_instance
119
def _unparse_url(self, path=None):
122
path = urllib.quote(path)
123
netloc = urllib.quote(self._host)
124
if self._username is not None:
125
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
126
if self._port is not None:
127
netloc = '%s:%d' % (netloc, self._port)
131
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
134
"""Return the ftplib.FTP instance for this object."""
135
if self._FTP_instance is not None:
136
return self._FTP_instance
139
self._FTP_instance = _find_FTP(self._host, self._port,
140
self._username, self._password,
142
return self._FTP_instance
143
except ftplib.error_perm, e:
144
raise errors.TransportError(msg="Error setting up connection: %s"
145
% str(e), orig_error=e)
147
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
148
"""Try to translate an ftplib.error_perm exception.
150
:param err: The error to translate into a bzr error
151
:param path: The path which had problems
152
:param extra: Extra information which can be included
153
:param unknown_exc: If None, we will just raise the original exception
154
otherwise we raise unknown_exc(path, extra=extra)
160
extra += ': ' + str(err)
161
if ('no such file' in s
162
or 'could not open' in s
163
or 'no such dir' in s
165
raise errors.NoSuchFile(path, extra=extra)
166
if ('file exists' in s):
167
raise errors.FileExists(path, extra=extra)
168
if ('not a directory' in s):
169
raise errors.PathError(path, extra=extra)
171
mutter('unable to understand error for path: %s: %s', path, err)
174
raise unknown_exc(path, extra=extra)
175
# TODO: jam 20060516 Consider re-raising the error wrapped in
176
# something like TransportError, but this loses the traceback
177
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
178
# to handle. Consider doing something like that here.
179
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
182
def should_cache(self):
183
"""Return True if the data pulled across should be cached locally.
187
def clone(self, offset=None):
188
"""Return a new FtpTransport with root at self.base + offset.
192
return FtpTransport(self.base, self._FTP_instance)
194
return FtpTransport(self.abspath(offset), self._FTP_instance)
196
def _abspath(self, relpath):
197
assert isinstance(relpath, basestring)
198
relpath = urlutils.unescape(relpath)
199
relpath_parts = relpath.split('/')
200
if len(relpath_parts) > 1:
201
if relpath_parts[0] == '':
202
raise ValueError("path %r within branch %r seems to be absolute"
203
% (relpath, self._path))
204
basepath = self._path.split('/')
205
if len(basepath) > 0 and basepath[-1] == '':
206
basepath = basepath[:-1]
207
for p in relpath_parts:
209
if len(basepath) == 0:
210
# In most filesystems, a request for the parent
211
# of root, just returns root.
214
elif p == '.' or p == '':
218
# Possibly, we could use urlparse.urljoin() here, but
219
# I'm concerned about when it chooses to strip the last
220
# portion of the path, and when it doesn't.
222
# XXX: It seems that ftplib does not handle Unicode paths
223
# at the same time, medusa won't handle utf8 paths
224
# So if we .encode(utf8) here, then we get a Server failure.
225
# while if we use str(), we get a UnicodeError, and the test suite
226
# just skips testing UnicodePaths.
227
return str('/'.join(basepath) or '/')
229
def abspath(self, relpath):
230
"""Return the full url to the given relative path.
231
This can be supplied with a string or a list
233
path = self._abspath(relpath)
234
return self._unparse_url(path)
236
def has(self, relpath):
237
"""Does the target location exist?"""
238
# FIXME jam 20060516 We *do* ask about directories in the test suite
239
# We don't seem to in the actual codebase
240
# XXX: I assume we're never asked has(dirname) and thus I use
241
# the FTP size command and assume that if it doesn't raise,
243
abspath = self._abspath(relpath)
246
mutter('FTP has check: %s => %s', relpath, abspath)
248
mutter("FTP has: %s", abspath)
250
except ftplib.error_perm, e:
251
if ('is a directory' in str(e).lower()):
252
mutter("FTP has dir: %s: %s", abspath, e)
254
mutter("FTP has not: %s: %s", abspath, e)
257
def get(self, relpath, decode=False, retries=0):
258
"""Get the file at the given relative path.
260
:param relpath: The relative path to the file
261
:param retries: Number of retries after temporary failures so far
264
We're meant to return a file-like object which bzr will
265
then read from. For now we do this via the magic of StringIO
267
# TODO: decode should be deprecated
269
mutter("FTP get: %s", self._abspath(relpath))
272
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
275
except ftplib.error_perm, e:
276
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
277
except ftplib.error_temp, e:
278
if retries > _number_of_retries:
279
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
280
% self.abspath(relpath),
283
warning("FTP temporary error: %s. Retrying.", str(e))
284
self._FTP_instance = None
285
return self.get(relpath, decode, retries+1)
287
if retries > _number_of_retries:
288
raise errors.TransportError("FTP control connection closed during GET %s."
289
% self.abspath(relpath),
292
warning("FTP control connection closed. Trying to reopen.")
293
time.sleep(_sleep_between_retries)
294
self._FTP_instance = None
295
return self.get(relpath, decode, retries+1)
297
def put(self, relpath, fp, mode=None, retries=0):
298
"""Copy the file-like or string object into the location.
300
:param relpath: Location to put the contents, relative to base.
301
:param fp: File-like or string object.
302
:param retries: Number of retries after temporary failures so far
305
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
307
abspath = self._abspath(relpath)
308
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
309
os.getpid(), random.randint(0,0x7FFFFFFF))
310
if not hasattr(fp, 'read'):
313
mutter("FTP put: %s", abspath)
316
f.storbinary('STOR '+tmp_abspath, fp)
317
f.rename(tmp_abspath, abspath)
318
except (ftplib.error_temp,EOFError), e:
319
warning("Failure during ftp PUT. Deleting temporary file.")
321
f.delete(tmp_abspath)
323
warning("Failed to delete temporary file on the"
324
" server.\nFile: %s", tmp_abspath)
327
except ftplib.error_perm, e:
328
self._translate_perm_error(e, abspath, extra='could not store')
329
except ftplib.error_temp, e:
330
if retries > _number_of_retries:
331
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
332
% self.abspath(relpath), orig_error=e)
334
warning("FTP temporary error: %s. Retrying.", str(e))
335
self._FTP_instance = None
336
self.put(relpath, fp, mode, retries+1)
338
if retries > _number_of_retries:
339
raise errors.TransportError("FTP control connection closed during PUT %s."
340
% self.abspath(relpath), orig_error=e)
342
warning("FTP control connection closed. Trying to reopen.")
343
time.sleep(_sleep_between_retries)
344
self._FTP_instance = None
345
self.put(relpath, fp, mode, retries+1)
347
def mkdir(self, relpath, mode=None):
348
"""Create a directory at the given path."""
349
abspath = self._abspath(relpath)
351
mutter("FTP mkd: %s", abspath)
354
except ftplib.error_perm, e:
355
self._translate_perm_error(e, abspath,
356
unknown_exc=errors.FileExists)
358
def rmdir(self, rel_path):
359
"""Delete the directory at rel_path"""
360
abspath = self._abspath(rel_path)
362
mutter("FTP rmd: %s", abspath)
365
except ftplib.error_perm, e:
366
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
368
def append(self, relpath, f, mode=None):
369
"""Append the text in the file-like object into the final
372
abspath = self._abspath(relpath)
373
if self.has(relpath):
374
ftp = self._get_FTP()
375
result = ftp.size(abspath)
379
mutter("FTP appe to %s", abspath)
380
self._try_append(relpath, f.read(), mode)
384
def _try_append(self, relpath, text, mode=None, retries=0):
385
"""Try repeatedly to append the given text to the file at relpath.
387
This is a recursive function. On errors, it will be called until the
388
number of retries is exceeded.
391
abspath = self._abspath(relpath)
392
mutter("FTP appe (try %d) to %s", retries, abspath)
393
ftp = self._get_FTP()
394
ftp.voidcmd("TYPE I")
395
cmd = "APPE %s" % abspath
396
conn = ftp.transfercmd(cmd)
400
self._setmode(relpath, mode)
402
except ftplib.error_perm, e:
403
self._translate_perm_error(e, abspath, extra='error appending',
404
unknown_exc=errors.NoSuchFile)
405
except ftplib.error_temp, e:
406
if retries > _number_of_retries:
407
raise errors.TransportError("FTP temporary error during APPEND %s." \
408
"Aborting." % abspath, orig_error=e)
410
warning("FTP temporary error: %s. Retrying.", str(e))
411
self._FTP_instance = None
412
self._try_append(relpath, text, mode, retries+1)
414
def _setmode(self, relpath, mode):
415
"""Set permissions on a path.
417
Only set permissions if the FTP server supports the 'SITE CHMOD'
421
mutter("FTP site chmod: setting permissions to %s on %s",
422
str(mode), self._abspath(relpath))
423
ftp = self._get_FTP()
424
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
426
except ftplib.error_perm, e:
427
# Command probably not available on this server
428
warning("FTP Could not set permissions to %s on %s. %s",
429
str(mode), self._abspath(relpath), str(e))
431
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
432
# to copy something to another machine. And you may be able
433
# to give it its own address as the 'to' location.
434
# So implement a fancier 'copy()'
436
def move(self, rel_from, rel_to):
437
"""Move the item at rel_from to the location at rel_to"""
438
abs_from = self._abspath(rel_from)
439
abs_to = self._abspath(rel_to)
441
mutter("FTP mv: %s => %s", abs_from, abs_to)
443
f.rename(abs_from, abs_to)
444
except ftplib.error_perm, e:
445
self._translate_perm_error(e, abs_from,
446
extra='unable to rename to %r' % (rel_to,),
447
unknown_exc=errors.PathError)
451
def delete(self, relpath):
452
"""Delete the item at relpath"""
453
abspath = self._abspath(relpath)
455
mutter("FTP rm: %s", abspath)
458
except ftplib.error_perm, e:
459
self._translate_perm_error(e, abspath, 'error deleting',
460
unknown_exc=errors.NoSuchFile)
463
"""See Transport.listable."""
466
def list_dir(self, relpath):
467
"""See Transport.list_dir."""
469
mutter("FTP nlst: %s", self._abspath(relpath))
471
basepath = self._abspath(relpath)
472
paths = f.nlst(basepath)
473
# If FTP.nlst returns paths prefixed by relpath, strip 'em
474
if paths and paths[0].startswith(basepath):
475
paths = [path[len(basepath)+1:] for path in paths]
476
# Remove . and .. if present, and return
477
return [path for path in paths if path not in (".", "..")]
478
except ftplib.error_perm, e:
479
self._translate_perm_error(e, relpath, extra='error with list_dir')
481
def iter_files_recursive(self):
482
"""See Transport.iter_files_recursive.
484
This is cargo-culted from the SFTP transport"""
485
mutter("FTP iter_files_recursive")
486
queue = list(self.list_dir("."))
488
relpath = urllib.quote(queue.pop(0))
489
st = self.stat(relpath)
490
if stat.S_ISDIR(st.st_mode):
491
for i, basename in enumerate(self.list_dir(relpath)):
492
queue.insert(i, relpath+"/"+basename)
496
def stat(self, relpath):
497
"""Return the stat information for a file."""
498
abspath = self._abspath(relpath)
500
mutter("FTP stat: %s", abspath)
502
return FtpStatResult(f, abspath)
503
except ftplib.error_perm, e:
504
self._translate_perm_error(e, abspath, extra='error w/ stat')
506
def lock_read(self, relpath):
507
"""Lock the given file for shared (read) access.
508
:return: A lock object, which should be passed to Transport.unlock()
510
# The old RemoteBranch ignore lock for reading, so we will
511
# continue that tradition and return a bogus lock object.
512
class BogusLock(object):
513
def __init__(self, path):
517
return BogusLock(relpath)
519
def lock_write(self, relpath):
520
"""Lock the given file for exclusive (write) access.
521
WARNING: many transports do not support this, so trying avoid using it
523
:return: A lock object, which should be passed to Transport.unlock()
525
return self.lock_read(relpath)
528
class FtpServer(Server):
529
"""Common code for SFTP server facilities."""
533
self._ftp_server = None
535
self._async_thread = None
540
"""Calculate an ftp url to this server."""
541
return 'ftp://foo:bar@localhost:%d/' % (self._port)
543
# def get_bogus_url(self):
544
# """Return a URL which cannot be connected to."""
545
# return 'ftp://127.0.0.1:1'
547
def log(self, message):
548
"""This is used by medusa.ftp_server to log connections, etc."""
549
self.logs.append(message)
554
raise RuntimeError('Must have medusa to run the FtpServer')
556
self._root = os.getcwdu()
557
self._ftp_server = _ftp_server(
558
authorizer=_test_authorizer(root=self._root),
560
port=0, # bind to a random port
562
logger_object=self # Use FtpServer.log() for messages
564
self._port = self._ftp_server.getsockname()[1]
565
# Don't let it loop forever, or handle an infinite number of requests.
566
# In this case it will run for 100s, or 1000 requests
567
self._async_thread = threading.Thread(target=asyncore.loop,
568
kwargs={'timeout':0.1, 'count':1000})
569
self._async_thread.setDaemon(True)
570
self._async_thread.start()
573
"""See bzrlib.transport.Server.tearDown."""
574
# have asyncore release the channel
575
self._ftp_server.del_channel()
577
self._async_thread.join()
582
_test_authorizer = None
586
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
589
import medusa.filesys
590
import medusa.ftp_server
596
class test_authorizer(object):
597
"""A custom Authorizer object for running the test suite.
599
The reason we cannot use dummy_authorizer, is because it sets the
600
channel to readonly, which we don't always want to do.
603
def __init__(self, root):
606
def authorize(self, channel, username, password):
607
"""Return (success, reply_string, filesystem)"""
609
return 0, 'No Medusa.', None
611
channel.persona = -1, -1
612
if username == 'anonymous':
613
channel.read_only = 1
615
channel.read_only = 0
617
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
620
class ftp_channel(medusa.ftp_server.ftp_channel):
621
"""Customized ftp channel"""
623
def log(self, message):
624
"""Redirect logging requests."""
625
mutter('_ftp_channel: %s', message)
627
def log_info(self, message, type='info'):
628
"""Redirect logging requests."""
629
mutter('_ftp_channel %s: %s', type, message)
631
def cmd_rnfr(self, line):
632
"""Prepare for renaming a file."""
633
self._renaming = line[1]
634
self.respond('350 Ready for RNTO')
635
# TODO: jam 20060516 in testing, the ftp server seems to
636
# check that the file already exists, or it sends
637
# 550 RNFR command failed
639
def cmd_rnto(self, line):
640
"""Rename a file based on the target given.
642
rnto must be called after calling rnfr.
644
if not self._renaming:
645
self.respond('503 RNFR required first.')
646
pfrom = self.filesystem.translate(self._renaming)
647
self._renaming = None
648
pto = self.filesystem.translate(line[1])
650
os.rename(pfrom, pto)
651
except (IOError, OSError), e:
652
# TODO: jam 20060516 return custom responses based on
653
# why the command failed
654
self.respond('550 RNTO failed: %s' % (e,))
656
self.respond('550 RNTO failed')
657
# For a test server, we will go ahead and just die
660
self.respond('250 Rename successful.')
662
def cmd_size(self, line):
663
"""Return the size of a file
665
This is overloaded to help the test suite determine if the
666
target is a directory.
669
if not self.filesystem.isfile(filename):
670
if self.filesystem.isdir(filename):
671
self.respond('550 "%s" is a directory' % (filename,))
673
self.respond('550 "%s" is not a file' % (filename,))
675
self.respond('213 %d'
676
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
678
def cmd_mkd(self, line):
679
"""Create a directory.
681
Overloaded because default implementation does not distinguish
682
*why* it cannot make a directory.
685
self.command_not_understood(''.join(line))
689
self.filesystem.mkdir (path)
690
self.respond ('257 MKD command successful.')
691
except (IOError, OSError), e:
692
self.respond ('550 error creating directory: %s' % (e,))
694
self.respond ('550 error creating directory.')
697
class ftp_server(medusa.ftp_server.ftp_server):
698
"""Customize the behavior of the Medusa ftp_server.
700
There are a few warts on the ftp_server, based on how it expects
704
ftp_channel_class = ftp_channel
706
def __init__(self, *args, **kwargs):
707
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
708
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
710
def log(self, message):
711
"""Redirect logging requests."""
712
mutter('_ftp_server: %s', message)
714
def log_info(self, message, type='info'):
715
"""Override the asyncore.log_info so we don't stipple the screen."""
716
mutter('_ftp_server %s: %s', type, message)
718
_test_authorizer = test_authorizer
719
_ftp_channel = ftp_channel
720
_ftp_server = ftp_server
725
def get_test_permutations():
726
"""Return the permutations to be used in testing."""
727
if not _setup_medusa():
728
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
731
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
