13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
"""Transport is an abstraction layer to handle file access.
84
84
def _get_transport_modules():
85
85
"""Return a list of the modules providing transports."""
87
for prefix, factory_list in transport_list_registry.iteritems():
87
for prefix, factory_list in transport_list_registry.items():
88
88
for factory in factory_list:
89
89
if hasattr(factory, "_module_name"):
90
90
modules.add(factory._module_name)
92
92
modules.add(factory._obj.__module__)
93
# Add chroot directly, because there is no handler registered for it.
93
# Add chroot and pathfilter directly, because there is no handler
94
95
modules.add('bzrlib.transport.chroot')
96
modules.add('bzrlib.transport.pathfilter')
95
97
result = list(modules)
100
102
class TransportListRegistry(registry.Registry):
101
103
"""A registry which simplifies tracking available Transports.
103
A registration of a new protocol requires two step:
105
A registration of a new protocol requires two steps:
104
106
1) register the prefix with the function register_transport( )
105
107
2) register the protocol provider with the function
106
108
register_transport_provider( ) ( and the "lazy" variant )
108
110
This is needed because:
109
a) a single provider can support multple protcol ( like the ftp
111
a) a single provider can support multiple protocols ( like the ftp
110
112
provider which supports both the ftp:// and the aftp:// protocols )
111
113
b) a single protocol can have multiple providers ( like the http://
112
114
protocol which is supported by both the urllib and pycurl provider )
272
274
from/to a storage location.
274
276
Most functions have a _multi variant, which allows you to queue up
275
multiple requests. They generally have a dumb base implementation
277
multiple requests. They generally have a dumb base implementation
276
278
which just iterates over the arguments, but smart Transport
277
279
implementations can do pipelining.
278
280
In general implementations should support having a generator or a list
324
326
def clone(self, offset=None):
325
327
"""Return a new Transport object, cloned from the current location,
326
using a subdirectory or parent directory. This allows connections
328
using a subdirectory or parent directory. This allows connections
327
329
to be pooled, rather than a new one needed for each subdir.
329
331
raise NotImplementedError(self.clone)
333
def create_prefix(self):
334
"""Create all the directories leading down to self.base."""
336
needed = [cur_transport]
337
# Recurse upwards until we can create a directory successfully
339
new_transport = cur_transport.clone('..')
340
if new_transport.base == cur_transport.base:
341
raise errors.BzrCommandError(
342
"Failed to create path prefix for %s."
343
% cur_transport.base)
345
new_transport.mkdir('.')
346
except errors.NoSuchFile:
347
needed.append(new_transport)
348
cur_transport = new_transport
349
except errors.FileExists:
353
# Now we only need to create child directories
355
cur_transport = needed.pop()
356
cur_transport.ensure_base()
331
358
def ensure_base(self):
332
359
"""Ensure that the directory this transport references exists.
367
394
raise NotImplementedError(self.external_url)
369
396
def _pump(self, from_file, to_file):
370
"""Most children will need to copy from one file-like
397
"""Most children will need to copy from one file-like
371
398
object or string to another one.
372
399
This just gives them something easy to call.
382
409
except TypeError: # We can't tell how many, because relpaths is a generator
412
def _report_activity(self, bytes, direction):
413
"""Notify that this transport has activity.
415
Implementations should call this from all methods that actually do IO.
416
Be careful that it's not called twice, if one method is implemented on
419
:param bytes: Number of bytes read or written.
420
:param direction: 'read' or 'write' or None.
422
ui.ui_factory.report_transport_activity(self, bytes, direction)
385
424
def _update_pb(self, pb, msg, count, total):
386
425
"""Update the progress bar based on the current count
387
426
and total available, total may be None if it was
439
478
t._combine_paths('/home/sarah', '/etc')
442
:param base_path: urlencoded path for the transport root; typically a
481
:param base_path: urlencoded path for the transport root; typically a
443
482
URL but need not contain scheme/host/etc.
444
483
:param relpath: relative url string for relative part of remote path.
445
484
:return: urlencoded string for final path.
472
511
"""Return the recommended page size for this transport.
474
513
This is potentially different for every path in a given namespace.
475
For example, local transports might use an operating system call to
514
For example, local transports might use an operating system call to
476
515
get the block size for a given path, which can vary due to mount
484
523
"""Return the local path portion from a given absolute path.
486
525
This default implementation is not suitable for filesystems with
487
aliasing, such as that given by symlinks, where a path may not
488
start with our base, but still be a relpath once aliasing is
526
aliasing, such as that given by symlinks, where a path may not
527
start with our base, but still be a relpath once aliasing is
491
530
# TODO: This might want to use bzrlib.osutils.relpath
504
543
raise errors.NotLocalUrl(self.abspath(relpath))
507
545
def has(self, relpath):
508
546
"""Does the file relpath exist?
510
548
Note that some transports MAY allow querying on directories, but this
511
is not part of the protocol. In other words, the results of
549
is not part of the protocol. In other words, the results of
512
550
t.has("a_directory_name") are undefined.
569
607
:param relpath: The relative path to the file
571
return self.get(relpath).read()
573
@deprecated_method(one_four)
574
def get_smart_client(self):
575
"""Return a smart client for this transport if possible.
577
A smart client doesn't imply the presence of a smart server: it implies
578
that the smart protocol can be tunnelled via this transport.
580
:raises NoSmartServer: if no smart server client is available.
582
raise errors.NoSmartServer(self.base)
609
f = self.get(relpath)
584
615
def get_smart_medium(self):
585
616
"""Return a smart client medium for this transport if possible.
592
623
raise errors.NoSmartMedium(self)
594
@deprecated_method(one_four)
595
def get_shared_medium(self):
596
"""Return a smart client shared medium for this transport if possible.
598
A smart medium doesn't imply the presence of a smart server: it implies
599
that the smart protocol can be tunnelled via this transport.
601
:raises NoSmartMedium: if no smart server medium is available.
603
raise errors.NoSmartMedium(self)
605
625
def readv(self, relpath, offsets, adjust_for_latency=False,
606
626
upper_limit=None):
607
627
"""Get parts of the file at the given relative path.
688
708
# Now that we've read some data, see if we can yield anything back
689
709
while cur_offset_and_size in data_map:
690
710
this_data = data_map.pop(cur_offset_and_size)
691
yield cur_offset_and_size[0], this_data
692
cur_offset_and_size = offset_stack.next()
711
this_offset = cur_offset_and_size[0]
713
cur_offset_and_size = offset_stack.next()
714
except StopIteration:
715
# Close the file handle as there will be no more data
716
# The handle would normally be cleaned up as this code goes
717
# out of scope, but as we are a generator, not all code
718
# will re-enter once we have consumed all the expected
720
# zip(range(len(requests)), readv(foo, requests))
721
# Will stop because the range is done, and not run the
722
# cleanup code for the readv().
724
cur_offset_and_size = None
725
yield this_offset, this_data
694
727
def _sort_expand_and_combine(self, offsets, upper_limit):
695
728
"""Helper for readv.
811
848
"""Get a list of file-like objects, one for each entry in relpaths.
813
850
:param relpaths: A list of relative paths.
814
:param pb: An optional ProgressBar for indicating percent done.
851
:param pb: An optional ProgressTask for indicating percent done.
815
852
:return: A list or generator of file-like objects
817
854
# TODO: Consider having this actually buffer the requests,
931
968
be synchronised with any internal buffering that may be present.
933
970
:param relpath: The relative path to the file.
934
:param mode: The mode for the newly created file,
971
:param mode: The mode for the newly created file,
935
972
None means just use the default
936
973
:return: A FileStream. FileStream objects have two methods, write() and
937
974
close(). There is no guarantee that data is committed to the file
980
1017
the supplied location.
982
1019
:param files: A set of (path, f) entries
983
:param pb: An optional ProgressBar for indicating percent done.
1020
:param pb: An optional ProgressTask for indicating percent done.
985
1022
return self._iterate_over(files, self.append_file, pb, 'append', expand=True)
987
1024
def copy(self, rel_from, rel_to):
988
1025
"""Copy the item at rel_from to the location at rel_to.
990
Override this for efficiency if a specific transport can do it
1027
Override this for efficiency if a specific transport can do it
991
1028
faster than this default implementation.
993
1030
self.put_file(rel_to, self.get(rel_from))
995
1032
def copy_multi(self, relpaths, pb=None):
996
1033
"""Copy a bunch of entries.
998
1035
:param relpaths: A list of tuples of the form [(from, to), (from, to),...]
1000
1037
# This is the non-pipelined implementation, so that
1018
1055
def copy_tree(self, from_relpath, to_relpath):
1019
1056
"""Copy a subtree from one relpath to another.
1021
If a faster implementation is available, specific transports should
1058
If a faster implementation is available, specific transports should
1024
1061
source = self.clone(from_relpath)
1025
self.mkdir(to_relpath)
1026
1062
target = self.clone(to_relpath)
1064
# create target directory with the same rwx bits as source.
1065
# use mask to ensure that bits other than rwx are ignored.
1066
stat = self.stat(from_relpath)
1067
target.mkdir('.', stat.st_mode & 0777)
1068
source.copy_tree_to_transport(target)
1070
def copy_tree_to_transport(self, to_transport):
1071
"""Copy a subtree from one transport to another.
1073
self.base is used as the source tree root, and to_transport.base
1074
is used as the target. to_transport.base must exist (and be a
1028
1078
directories = ['.']
1029
1079
while directories:
1030
1080
dir = directories.pop()
1033
for path in source.list_dir(dir):
1082
to_transport.mkdir(dir)
1083
for path in self.list_dir(dir):
1034
1084
path = dir + '/' + path
1035
stat = source.stat(path)
1085
stat = self.stat(path)
1036
1086
if S_ISDIR(stat.st_mode):
1037
1087
directories.append(path)
1039
1089
files.append(path)
1040
source.copy_to(files, target)
1090
self.copy_to(files, to_transport)
1042
1092
def rename(self, rel_from, rel_to):
1043
1093
"""Rename a file or directory.
1077
1127
def move_multi(self, relpaths, pb=None):
1078
1128
"""Move a bunch of entries.
1080
1130
:param relpaths: A list of tuples of the form [(from1, to1), (from2, to2),...]
1082
1132
return self._iterate_over(relpaths, self.move, pb, 'move', expand=True)
1135
1185
NOTE: This returns an object with fields such as 'st_size'. It MAY
1136
1186
or MAY NOT return the literal result of an os.stat() call, so all
1137
1187
access should be via named fields.
1138
ALSO NOTE: Stats of directories may not be supported on some
1188
ALSO NOTE: Stats of directories may not be supported on some
1141
1191
raise NotImplementedError(self.stat)
1156
1206
count = self._iterate_over(relpaths, gather, pb, 'stat', expand=False)
1209
def readlink(self, relpath):
1210
"""Return a string representing the path to which the symbolic link points."""
1211
raise errors.TransportNotPossible("Dereferencing symlinks is not supported on %s" % self)
1213
def hardlink(self, source, link_name):
1214
"""Create a hardlink pointing to source named link_name."""
1215
raise errors.TransportNotPossible("Hard links are not supported on %s" % self)
1217
def symlink(self, source, link_name):
1218
"""Create a symlink pointing to source named link_name."""
1219
raise errors.TransportNotPossible("Symlinks are not supported on %s" % self)
1159
1221
def listable(self):
1160
1222
"""Return True if this store supports listing."""
1161
1223
raise NotImplementedError(self.listable)
1205
1267
"""Return true if this transport can store and retrieve unix modebits.
1207
1269
(For example, 0700 to make a directory owner-private.)
1209
Note: most callers will not want to switch on this, but should rather
1271
Note: most callers will not want to switch on this, but should rather
1210
1272
just try and set permissions and let them be either stored or not.
1211
1273
This is intended mainly for the use of the test suite.
1213
Warning: this is not guaranteed to be accurate as sometimes we can't
1275
Warning: this is not guaranteed to be accurate as sometimes we can't
1214
1276
be sure: for example with vfat mounted on unix, or a windows sftp
1216
1278
# TODO: Perhaps return a e.g. TransportCharacteristics that can answer
1223
1285
# should be asked to ConnectedTransport only.
1288
def _redirected_to(self, source, target):
1289
"""Returns a transport suitable to re-issue a redirected request.
1291
:param source: The source url as returned by the server.
1292
:param target: The target url as returned by the server.
1294
The redirection can be handled only if the relpath involved is not
1295
renamed by the redirection.
1297
:returns: A transport or None.
1299
# This returns None by default, meaning the transport can't handle the
1227
1305
class _SharedConnection(object):
1228
1306
"""A connection shared between several transports."""
1300
1378
def _split_url(url):
1302
Extract the server address, the credentials and the path from the url.
1304
user, password, host and path should be quoted if they contain reserved
1307
:param url: an quoted url
1309
:return: (scheme, user, password, host, port, path) tuple, all fields
1312
if isinstance(url, unicode):
1313
raise errors.InvalidURL('should be ascii:\n%r' % url)
1314
url = url.encode('utf-8')
1315
(scheme, netloc, path, params,
1316
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
1317
user = password = host = port = None
1319
user, host = netloc.rsplit('@', 1)
1321
user, password = user.split(':', 1)
1322
password = urllib.unquote(password)
1323
user = urllib.unquote(user)
1328
host, port = host.rsplit(':', 1)
1332
raise errors.InvalidURL('invalid port number %s in url:\n%s' %
1335
raise errors.InvalidURL('Host empty in: %s' % url)
1337
host = urllib.unquote(host)
1338
path = urllib.unquote(path)
1340
return (scheme, user, password, host, port, path)
1379
return urlutils.parse_url(url)
1343
1382
def _unsplit_url(scheme, user, password, host, port, path):
1369
1408
netloc = '%s@%s' % (urllib.quote(user), netloc)
1370
1409
if port is not None:
1371
1410
netloc = '%s:%d' % (netloc, port)
1372
path = urllib.quote(path)
1411
path = urlutils.escape(path)
1373
1412
return urlparse.urlunparse((scheme, netloc, path, None, None, None))
1375
1414
def relpath(self, abspath):
1534
1573
def convert_path_to_url(base, error_str):
1535
1574
m = _urlRE.match(base)
1537
# This looks like a URL, but we weren't able to
1576
# This looks like a URL, but we weren't able to
1538
1577
# instantiate it as such raise an appropriate error
1539
1578
# FIXME: we have a 'error_str' unused and we use last_err below
1540
1579
raise errors.UnsupportedProtocol(base, last_err)
1561
1600
possible_transports.append(t_same_connection)
1562
1601
return t_same_connection
1564
for proto, factory_list in transport_list_registry.iteritems():
1603
for proto, factory_list in transport_list_registry.items():
1565
1604
if proto is not None and base.startswith(proto):
1566
1605
transport, last_err = _try_transport_factories(base, factory_list)
1606
1645
:param action: A callable, what the caller want to do while catching
1608
1647
:param transport: The initial transport used.
1609
:param redirected: A callable receiving the redirected transport and the
1648
:param redirected: A callable receiving the redirected transport and the
1610
1649
RedirectRequested exception.
1612
1651
:return: Whatever 'action' returns
1640
1679
class Server(object):
1641
1680
"""A Transport Server.
1643
The Server interface provides a server for a given transport. We use
1644
these servers as loopback testing tools. For any given transport the
1645
Servers it provides must either allow writing, or serve the contents
1646
of os.getcwdu() at the time setUp is called.
1648
Note that these are real servers - they must implement all the things
1649
that we want bzr transports to take advantage of.
1682
The Server interface provides a server for a given transport type.
1685
def start_server(self):
1653
1686
"""Setup the server to service requests."""
1688
def stop_server(self):
1656
1689
"""Remove the server and cleanup any resources it owns."""
1659
"""Return a url for this server.
1661
If the transport does not represent a disk directory (i.e. it is
1662
a database like svn, or a memory only transport, it should return
1663
a connection to a newly established resource for this Server.
1664
Otherwise it should return a url that will provide access to the path
1665
that was os.getcwdu() when setUp() was called.
1667
Subsequent calls will return the same resource.
1669
raise NotImplementedError
1671
def get_bogus_url(self):
1672
"""Return a url for this protocol, that will fail to connect.
1674
This may raise NotImplementedError to indicate that this server cannot
1677
raise NotImplementedError
1680
1692
# None is the default transport, for things with no url scheme
1681
1693
register_transport_proto('file://',
1713
1725
help="Read-only access of branches exported on the web.")
1714
1726
register_transport_proto('https://',
1715
1727
help="Read-only access of branches exported on the web using SSL.")
1728
# The default http implementation is urllib, but https is pycurl if available
1729
register_lazy_transport('http://', 'bzrlib.transport.http._pycurl',
1716
1731
register_lazy_transport('http://', 'bzrlib.transport.http._urllib',
1717
1732
'HttpTransport_urllib')
1718
1733
register_lazy_transport('https://', 'bzrlib.transport.http._urllib',
1719
1734
'HttpTransport_urllib')
1720
register_lazy_transport('http://', 'bzrlib.transport.http._pycurl',
1722
1735
register_lazy_transport('https://', 'bzrlib.transport.http._pycurl',
1723
1736
'PyCurlTransport')
1727
1740
register_transport_proto('aftp://', help="Access using active FTP.")
1728
1741
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport')
1730
# Default to trying GSSAPI authentication (if the kerberos module is available)
1731
register_transport_proto('ftp+gssapi://', register_netloc=True)
1732
register_lazy_transport('ftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
1733
'GSSAPIFtpTransport')
1734
register_transport_proto('aftp+gssapi://', register_netloc=True)
1735
register_lazy_transport('aftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
1736
'GSSAPIFtpTransport')
1737
register_transport_proto('ftp+nogssapi://', register_netloc=True)
1738
register_transport_proto('aftp+nogssapi://', register_netloc=True)
1740
register_lazy_transport('ftp://', 'bzrlib.transport.ftp._gssapi',
1741
'GSSAPIFtpTransport')
1742
register_lazy_transport('aftp://', 'bzrlib.transport.ftp._gssapi',
1743
'GSSAPIFtpTransport')
1744
register_lazy_transport('ftp+nogssapi://', 'bzrlib.transport.ftp',
1746
register_lazy_transport('aftp+nogssapi://', 'bzrlib.transport.ftp',
1745
kerberos_available = True
1747
kerberos_available = False
1749
if kerberos_available:
1750
# Default to trying GSSAPI authentication (if the kerberos module is
1752
register_transport_proto('ftp+gssapi://', register_netloc=True)
1753
register_lazy_transport('ftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
1754
'GSSAPIFtpTransport')
1755
register_transport_proto('aftp+gssapi://', register_netloc=True)
1756
register_lazy_transport('aftp+gssapi://', 'bzrlib.transport.ftp._gssapi',
1757
'GSSAPIFtpTransport')
1758
register_transport_proto('ftp+nogssapi://', register_netloc=True)
1759
register_transport_proto('aftp+nogssapi://', register_netloc=True)
1761
register_lazy_transport('ftp://', 'bzrlib.transport.ftp._gssapi',
1762
'GSSAPIFtpTransport')
1763
register_lazy_transport('aftp://', 'bzrlib.transport.ftp._gssapi',
1764
'GSSAPIFtpTransport')
1765
register_lazy_transport('ftp+nogssapi://', 'bzrlib.transport.ftp',
1767
register_lazy_transport('aftp+nogssapi://', 'bzrlib.transport.ftp',
1749
1770
register_transport_proto('memory://')
1750
1771
register_lazy_transport('memory://', 'bzrlib.transport.memory',
1819
1840
register_netloc=True)
1820
1841
register_lazy_transport('bzr+ssh://', 'bzrlib.transport.remote',
1821
1842
'RemoteSSHTransport')
1844
register_transport_proto('ssh:')
1845
register_lazy_transport('ssh:', 'bzrlib.transport.remote',
1846
'HintingSSHTransport')
1849
transport_server_registry = registry.Registry()
1850
transport_server_registry.register_lazy('bzr', 'bzrlib.smart.server',
1851
'serve_bzr', help="The Bazaar smart server protocol over TCP. (default port: 4155)")
1852
transport_server_registry.default_key = 'bzr'
added
removed
bzrlib/transport/__init__.py
