/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar

« back to all changes in this revision

Viewing changes to bzrlib/transport/__init__.py

Better readv coalescing, now with test, and progress during knit index reading.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/transport/__init__.py
20
20
"""
21
21
 
22
22
import errno
 
23
from collections import deque
23
24
from copy import deepcopy
24
25
from stat import *
25
26
import sys
250
251
        :offsets: A list of (offset, size) tuples.
251
252
        :return: A list or generator of (offset, data) tuples
252
253
        """
 
254
        def do_combined_read(combined_offsets):
 
255
            total_size = 0
 
256
            for offset, size in combined_offsets:
 
257
                total_size += size
 
258
            mutter('readv coalesced %d reads.', len(combined_offsets))
 
259
            offset = combined_offsets[0][0]
 
260
            fp.seek(offset)
 
261
            data = fp.read(total_size)
 
262
            pos = 0
 
263
            for offset, size in combined_offsets:
 
264
                yield offset, data[pos:pos + size]
 
265
                pos += size
 
266
 
253
267
        if not len(offsets):
254
268
            return
255
269
        fp = self.get(relpath)
256
 
        # coalesce records
257
 
        new_offsets = [list(offsets[0])]
258
 
        for offset, size in offsets[1:]:
259
 
            if offset == new_offsets[-1][0] + new_offsets[-1][1]:
260
 
                new_offsets[-1][1] += size
 
270
        pending_offsets = deque(offsets)
 
271
        combined_offsets = []
 
272
        while len(pending_offsets):
 
273
            offset, size = pending_offsets.popleft()
 
274
            if not combined_offsets:
 
275
                combined_offsets = [[offset, size]]
261
276
            else:
262
 
                new_offsets.append([offset, size])
263
 
        mutter('readv coalesced %d reads to %d', len(offsets), len(new_offsets))
264
 
        for offset, size in new_offsets:
265
 
            fp.seek(offset)
266
 
            yield offset, fp.read(size)
 
277
                if combined_offsets[-1][0] + combined_offsets[-1][1] == offset:
 
278
                    # combatible offset:
 
279
                    combined_offsets.append([offset, size])
 
280
                else:
 
281
                    # incompatible, issue a read and yield
 
282
                    pending_offsets.appendleft((offset, size))
 
283
                    for result in do_combined_read(combined_offsets):
 
284
                        yield result
 
285
                    combined_offsets = []
 
286
        # whatever is left is a single coalesced request
 
287
        if len(combined_offsets):
 
288
            for result in do_combined_read(combined_offsets):
 
289
                yield result
267
290
 
268
291
    def get_multi(self, relpaths, pb=None):
269
292
        """Get a list of file-like objects, one for each entry in relpaths.