1
# Copyright (C) 2005, 2006 by Canonical Ltd
2
# Written by Robert Collins <robert.collins@canonical.com>
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU General Public License for more details.
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
"""Bzrlib specific gzip tunings. We plan to feed these to the upstream gzip."""
20
# make GzipFile faster:
22
from gzip import U32, LOWU32, FEXTRA, FCOMMENT, FNAME, FHCRC
27
__all__ = ["GzipFile"]
30
class GzipFile(gzip.GzipFile):
31
"""Knit tuned version of GzipFile.
33
This is based on the following lsprof stats:
34
python 2.4 stock GzipFile write:
35
58971 0 5644.3090 2721.4730 gzip:193(write)
36
+58971 0 1159.5530 1159.5530 +<built-in method compress>
37
+176913 0 987.0320 987.0320 +<len>
38
+58971 0 423.1450 423.1450 +<zlib.crc32>
39
+58971 0 353.1060 353.1060 +<method 'write' of 'cStringIO.
42
58971 0 4477.2590 2103.1120 bzrlib.knit:1250(write)
43
+58971 0 1297.7620 1297.7620 +<built-in method compress>
44
+58971 0 406.2160 406.2160 +<zlib.crc32>
45
+58971 0 341.9020 341.9020 +<method 'write' of 'cStringIO.
47
+58971 0 328.2670 328.2670 +<len>
50
Yes, its only 1.6 seconds, but they add up.
53
def _add_read_data(self, data):
55
# temp var for len(data) and switch to +='s.
58
self.crc = zlib.crc32(data, self.crc)
60
self.extrasize += len_data
63
def _read(self, size=1024):
64
# various optimisations:
65
# reduces lsprof count from 2500 to
66
# 8337 calls in 1272, 365 internal
67
if self.fileobj is None:
68
raise EOFError, "Reached EOF"
71
# If the _new_member flag is set, we have to
72
# jump to the next member, if there is one.
74
# First, check if we're at the end of the file;
75
# if so, it's time to stop; no more members to read.
76
next_header_bytes = self.fileobj.read(10)
77
if next_header_bytes == '':
78
raise EOFError, "Reached EOF"
81
self._read_gzip_header(next_header_bytes)
82
self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
83
self._new_member = False
85
# Read a chunk of data from the file
86
buf = self.fileobj.read(size)
88
# If the EOF has been reached, flush the decompression object
89
# and mark this object as finished.
92
self._add_read_data(self.decompress.flush())
93
assert len(self.decompress.unused_data) >= 8, "what does flush do?"
95
# tell the driving read() call we have stuffed all the data
97
raise EOFError, 'Reached EOF'
99
self._add_read_data(self.decompress.decompress(buf))
101
if self.decompress.unused_data != "":
102
# Ending case: we've come to the end of a member in the file,
103
# so seek back to the start of the data for the next member which
104
# is the length of the decompress objects unused data - the first
105
# 8 bytes for the end crc and size records.
107
# so seek back to the start of the unused data, finish up
108
# this member, and read a new gzip header.
109
# (The number of bytes to seek back is the length of the unused
110
# data, minus 8 because those 8 bytes are part of this member.
111
seek_length = len (self.decompress.unused_data) - 8
113
assert seek_length > 0
114
self.fileobj.seek(-seek_length, 1)
116
# Check the CRC and file size, and set the flag so we read
117
# a new member on the next call
119
self._new_member = True
122
"""tuned to reduce function calls and eliminate file seeking:
124
reduces lsprof count from 800 to 288
126
avoid U32 call by using struct format L
129
# We've read to the end of the file, so we should have 8 bytes of
130
# unused data in the decompressor. If we dont, there is a corrupt file.
131
# We use these 8 bytes to calculate the CRC and the recorded file size.
132
# We then check the that the computed CRC and size of the
133
# uncompressed data matches the stored values. Note that the size
134
# stored is the true file size mod 2**32.
135
crc32, isize = struct.unpack("<LL", self.decompress.unused_data[0:8])
136
# note that isize is unsigned - it can exceed 2GB
137
if crc32 != U32(self.crc):
138
raise IOError, "CRC check failed"
139
elif isize != LOWU32(self.size):
140
raise IOError, "Incorrect length of data produced"
142
def _read_gzip_header(self, bytes=None):
143
"""Supply bytes if the minimum header size is already read.
145
:param bytes: 10 bytes of header data.
147
"""starting cost: 300 in 3998
148
15998 reads from 3998 calls
152
bytes = self.fileobj.read(10)
154
if magic != '\037\213':
155
raise IOError, 'Not a gzipped file'
156
method = ord(bytes[2:3])
158
raise IOError, 'Unknown compression method'
159
flag = ord(bytes[3:4])
160
# modtime = self.fileobj.read(4) (bytes [4:8])
161
# extraflag = self.fileobj.read(1) (bytes[8:9])
162
# os = self.fileobj.read(1) (bytes[9:10])
163
# self.fileobj.read(6)
166
# Read & discard the extra field, if present
167
xlen = ord(self.fileobj.read(1))
168
xlen = xlen + 256*ord(self.fileobj.read(1))
169
self.fileobj.read(xlen)
171
# Read and discard a null-terminated string containing the filename
173
s = self.fileobj.read(1)
174
if not s or s=='\000':
177
# Read and discard a null-terminated string containing a comment
179
s = self.fileobj.read(1)
180
if not s or s=='\000':
183
self.fileobj.read(2) # Read & discard the 16-bit header CRC
185
def readline(self, size=-1):
186
"""Tuned to remove buffer length calls in _unread and...
188
also removes multiple len(c) calls, inlines _unread,
189
total savings - lsprof 5800 to 5300
192
8176 calls to read() in 1684
193
changing the min chunk size to 200 halved all the cache misses
194
leading to a drop to:
196
4168 call to read() in 1646
197
- i.e. just reduced the function call overhead. May be worth
200
if size < 0: size = sys.maxint
202
readsize = min(200, size) # Read from the file in small chunks
205
return "".join(bufs) # Return resulting line
208
c = self.read(readsize)
209
# number of bytes read
213
# We set i=size to break out of the loop under two
214
# conditions: 1) there's no newline, and the chunk is
215
# larger than size, or 2) there is a newline, but the
216
# resulting line would be longer than 'size'.
217
if i==-1 and len_c > size: i=size-1
218
elif size <= i: i = size -1
220
if i >= 0 or c == '':
221
# if i>= 0 we have a newline or have triggered the above
222
# if size is not None condition.
223
# if c == '' its EOF.
224
bufs.append(c[:i+1]) # Add portion of last chunk
225
# -- inlined self._unread --
226
## self._unread(c[i+1:], len_c - i) # Push back rest of chunk
227
self.extrabuf = c[i+1:] + self.extrabuf
228
self.extrasize = len_c - i + self.extrasize
229
self.offset -= len_c - i
230
# -- end inlined self._unread --
231
return ''.join(bufs) # Return resulting line
233
# Append chunk to list, decrease 'size',
236
readsize = min(size, readsize * 2)
238
def readlines(self, sizehint=0):
239
# optimise to avoid all the buffer manipulation
240
# lsprof changed from:
241
# 4168 calls in 5472 with 32000 calls to readline()
244
# Negative numbers result in reading all the lines
247
content = self.read(sizehint)
248
return content.splitlines(True)
250
def _unread(self, buf, len_buf=None):
251
"""tuned to remove unneeded len calls.
253
because this is such an inner routine in readline, and readline is
254
in many inner loops, this has been inlined into readline().
256
The len_buf parameter combined with the reduction in len calls dropped
257
the lsprof ms count for this routine on my test data from 800 to 200 -
262
self.extrabuf = buf + self.extrabuf
263
self.extrasize = len_buf + self.extrasize
264
self.offset -= len_buf
266
def write(self, data):
267
if self.mode != gzip.WRITE:
269
raise IOError(errno.EBADF, "write() on read-only GzipFile object")
271
if self.fileobj is None:
272
raise ValueError, "write() on closed GzipFile object"
275
self.size = self.size + data_len
276
self.crc = zlib.crc32(data, self.crc)
277
self.fileobj.write( self.compress.compress(data) )
278
self.offset += data_len
280
def writelines(self, lines):
281
# profiling indicated a significant overhead
282
# calling write for each line.
283
# this batch call is a lot faster :).
284
# (4 seconds to 1 seconds for the sample upgrades I was testing).
285
self.write(''.join(lines))
added
removed
bzrlib/tuned_gzip.py
