125
125
self.get(key).insert(0, registry._ObjectGetter(obj))
127
127
def register_lazy_transport_provider(self, key, module_name, member_name):
128
self.get(key).insert(0,
128
self.get(key).insert(0,
129
129
registry._LazyObjectGetter(module_name, member_name))
131
def register_transport(self, key, help=None, info=None):
132
self.register(key, [], help, info)
131
def register_transport(self, key, help=None, default_port=None):
132
self.register(key, [], help, default_port)
134
134
def set_default_transport(self, key=None):
135
135
"""Return either 'key' or the default key if key is None"""
136
136
self._default_key = key
139
transport_list_registry = TransportListRegistry( )
142
def register_transport_proto(prefix, help=None, info=None):
143
transport_list_registry.register_transport(prefix, help, info)
138
def get_default_port(self, scheme):
139
"""Return the registered default port for this protocol scheme."""
141
return self.get_info(scheme + '://')
146
transport_list_registry = TransportListRegistry()
149
def register_transport_proto(prefix, help=None, info=None, default_port=None):
150
transport_list_registry.register_transport(prefix, help, default_port)
146
153
def register_lazy_transport(prefix, module, classname):
636
643
raise errors.NoSmartMedium(self)
638
def readv(self, relpath, offsets):
639
"""Get parts of the file at the given relative path.
641
:offsets: A list of (offset, size) tuples.
645
def readv(self, relpath, offsets, adjust_for_latency=False,
647
"""Get parts of the file at the given relative path.
649
:param relpath: The path to read data from.
650
:param offsets: A list of (offset, size) tuples.
651
:param adjust_for_latency: Adjust the requested offsets to accomdate
652
transport latency. This may re-order the offsets, expand them to
653
grab adjacent data when there is likely a high cost to requesting
654
data relative to delivering it.
655
:param upper_limit: When adjust_for_latency is True setting upper_limit
656
allows the caller to tell the transport about the length of the
657
file, so that requests are not issued for ranges beyond the end of
658
the file. This matters because some servers and/or transports error
659
in such a case rather than just satisfying the available ranges.
660
upper_limit should always be provided when adjust_for_latency is
661
True, and should be the size of the file in bytes.
662
:return: A list or generator of (offset, data) tuples
664
if adjust_for_latency:
665
# Design note: We may wish to have different algorithms for the
666
# expansion of the offsets per-transport. E.g. for local disk to
667
# use page-aligned expansion. If that is the case consider the following structure:
668
# - a test that transport.readv uses self._offset_expander or some similar attribute, to do the expansion
669
# - a test for each transport that it has some known-good offset expander
670
# - unit tests for each offset expander
671
# - a set of tests for the offset expander interface, giving
672
# baseline behaviour (which the current transport
673
# adjust_for_latency tests could be repurposed to).
674
offsets = self._sort_expand_and_combine(offsets, upper_limit)
675
return self._readv(relpath, offsets)
677
def _readv(self, relpath, offsets):
678
"""Get parts of the file at the given relative path.
680
:param relpath: The path to read.
681
:param offsets: A list of (offset, size) tuples.
642
682
:return: A list or generator of (offset, data) tuples
688
728
yield cur_offset_and_size[0], this_data
689
729
cur_offset_and_size = offset_stack.next()
731
def _sort_expand_and_combine(self, offsets, upper_limit):
734
:param offsets: A readv vector - (offset, length) tuples.
735
:param upper_limit: The highest byte offset that may be requested.
736
:return: A readv vector that will read all the regions requested by
737
offsets, in start-to-end order, with no duplicated regions,
738
expanded by the transports recommended page size.
740
offsets = sorted(offsets)
741
# short circuit empty requests
742
if len(offsets) == 0:
744
# Quick thunk to stop this function becoming a generator
745
# itself, rather we return a generator that has nothing to
749
return empty_yielder()
750
# expand by page size at either end
751
maximum_expansion = self.recommended_page_size()
753
for offset, length in offsets:
754
expansion = maximum_expansion - length
756
# we're asking for more than the minimum read anyway.
758
reduction = expansion / 2
759
new_offset = offset - reduction
760
new_length = length + expansion
762
# don't ask for anything < 0
764
if (upper_limit is not None and
765
new_offset + new_length > upper_limit):
766
new_length = upper_limit - new_offset
767
new_offsets.append((new_offset, new_length))
768
# combine the expanded offsets
770
current_offset, current_length = new_offsets[0]
771
current_finish = current_length + current_offset
772
for offset, length in new_offsets[1:]:
773
finish = offset + length
774
if offset > current_finish:
775
# there is a gap, output the current accumulator and start
776
# a new one for the region we're examining.
777
offsets.append((current_offset, current_length))
778
current_offset = offset
779
current_length = length
780
current_finish = finish
782
if finish > current_finish:
783
# extend the current accumulator to the end of the region
785
current_finish = finish
786
current_length = finish - current_offset
787
offsets.append((current_offset, current_length))
692
791
def _coalesce_offsets(offsets, limit, fudge_factor):
693
792
"""Yield coalesced offsets.
1286
1389
# have one so that it doesn't get accidentally
1288
1391
netloc = '%s@%s' % (urllib.quote(user), netloc)
1289
if port is not None:
1392
if (port is not None and
1393
port != transport_list_registry.get_default_port(scheme)):
1394
# Include the port in the netloc (unless it's the same as the
1395
# default, in which case we omit it as it is redundant).
1290
1396
netloc = '%s:%d' % (netloc, port)
1291
1397
path = urllib.quote(path)
1292
1398
return urlparse.urlunparse((scheme, netloc, path, None, None, None))
1595
1701
raise NotImplementedError
1598
class TransportLogger(object):
1599
"""Adapt a transport to get clear logging data on api calls.
1601
Feel free to extend to log whatever calls are of interest.
1604
def __init__(self, adapted):
1605
self._adapted = adapted
1608
def get(self, name):
1609
self._calls.append((name,))
1610
return self._adapted.get(name)
1612
def __getattr__(self, name):
1613
"""Thunk all undefined access through to self._adapted."""
1614
# raise AttributeError, name
1615
return getattr(self._adapted, name)
1617
def readv(self, name, offsets):
1618
self._calls.append((name, offsets))
1619
return self._adapted.readv(name, offsets)
1622
1704
# None is the default transport, for things with no url scheme
1623
1705
register_transport_proto('file://',
1624
1706
help="Access using the standard filesystem (default)")
1625
1707
register_lazy_transport('file://', 'bzrlib.transport.local', 'LocalTransport')
1626
1708
transport_list_registry.set_default_transport("file://")
1710
# Note that sftp:// has no default_port, because the user's ~/.ssh/config
1711
# can set it to arbitrary values based on hostname.
1628
1712
register_transport_proto('sftp://',
1629
1713
help="Access using SFTP (most SSH servers provide SFTP).")
1630
1714
register_lazy_transport('sftp://', 'bzrlib.transport.sftp', 'SFTPTransport')
1631
1715
# Decorated http transport
1632
1716
register_transport_proto('http+urllib://',
1633
1717
# help="Read-only access of branches exported on the web."
1635
1719
register_lazy_transport('http+urllib://', 'bzrlib.transport.http._urllib',
1636
1720
'HttpTransport_urllib')
1637
1721
register_transport_proto('https+urllib://',
1638
1722
# help="Read-only access of branches exported on the web using SSL."
1640
1724
register_lazy_transport('https+urllib://', 'bzrlib.transport.http._urllib',
1641
1725
'HttpTransport_urllib')
1642
1726
register_transport_proto('http+pycurl://',
1643
1727
# help="Read-only access of branches exported on the web."
1645
1729
register_lazy_transport('http+pycurl://', 'bzrlib.transport.http._pycurl',
1646
1730
'PyCurlTransport')
1647
1731
register_transport_proto('https+pycurl://',
1648
1732
# help="Read-only access of branches exported on the web using SSL."
1650
1734
register_lazy_transport('https+pycurl://', 'bzrlib.transport.http._pycurl',
1651
1735
'PyCurlTransport')
1652
1736
# Default http transports (last declared wins (if it can be imported))
1653
1737
register_transport_proto('http://',
1654
help="Read-only access of branches exported on the web.")
1738
help="Read-only access of branches exported on the web.",
1655
1740
register_transport_proto('https://',
1656
help="Read-only access of branches exported on the web using SSL.")
1741
help="Read-only access of branches exported on the web using SSL.",
1657
1743
register_lazy_transport('http://', 'bzrlib.transport.http._urllib',
1658
1744
'HttpTransport_urllib')
1659
1745
register_lazy_transport('https://', 'bzrlib.transport.http._urllib',
1662
1748
register_lazy_transport('https://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport')
1664
1750
register_transport_proto('ftp://',
1665
help="Access using passive FTP.")
1751
help="Access using passive FTP.",
1666
1753
register_lazy_transport('ftp://', 'bzrlib.transport.ftp', 'FtpTransport')
1667
1754
register_transport_proto('aftp://',
1668
help="Access using active FTP.")
1755
help="Access using active FTP.",
1669
1757
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport')
1671
1759
register_transport_proto('memory://')
1672
1760
register_lazy_transport('memory://', 'bzrlib.transport.memory', 'MemoryTransport')
1762
# chroots cannot be implicitly accessed, they must be explicitly created:
1673
1763
register_transport_proto('chroot+')
1675
1765
register_transport_proto('readonly+',
1692
1785
'bzrlib.transport.fakevfat',
1693
1786
'FakeVFATTransportDecorator')
1694
1787
register_transport_proto('bzr://',
1695
help="Fast access using the Bazaar smart server.")
1788
help="Fast access using the Bazaar smart server.",
1697
1791
register_lazy_transport('bzr://',
1698
1792
'bzrlib.transport.remote',
1699
1793
'RemoteTCPTransport')
1700
1794
register_transport_proto('bzr+http://',
1701
1795
# help="Fast access using the Bazaar smart server over HTTP."
1797
register_lazy_transport('bzr+http://',
1798
'bzrlib.transport.remote',
1799
'RemoteHTTPTransport')
1800
register_transport_proto('bzr+https://',
1801
# help="Fast access using the Bazaar smart server over HTTPS."
1703
register_lazy_transport('bzr+http://',
1803
register_lazy_transport('bzr+https://',
1704
1804
'bzrlib.transport.remote',
1705
1805
'RemoteHTTPTransport')
1806
# Note that bzr+ssh:// has no default_port, because the user's ~/.ssh/config
1807
# can set it to arbitrary values based on hostname.
1706
1808
register_transport_proto('bzr+ssh://',
1707
1809
help="Fast access using the Bazaar smart server over SSH.")
1708
1810
register_lazy_transport('bzr+ssh://',