1
# Copyright (C) 2007, 2009 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Container format for Bazaar data.
19
"Containers" and "records" are described in
20
doc/developers/container-format.txt.
23
from cStringIO import StringIO
26
from bzrlib import errors
29
FORMAT_ONE = "Bazaar pack format 1 (introduced in 0.18)"
32
_whitespace_re = re.compile('[\t\n\x0b\x0c\r ]')
35
def _check_name(name):
36
"""Do some basic checking of 'name'.
38
At the moment, this just checks that there are no whitespace characters in a
41
:raises InvalidRecordError: if name is not valid.
42
:seealso: _check_name_encoding
44
if _whitespace_re.search(name) is not None:
45
raise errors.InvalidRecordError("%r is not a valid name." % (name,))
48
def _check_name_encoding(name):
49
"""Check that 'name' is valid UTF-8.
51
This is separate from _check_name because UTF-8 decoding is relatively
52
expensive, and we usually want to avoid it.
54
:raises InvalidRecordError: if name is not valid UTF-8.
58
except UnicodeDecodeError, e:
59
raise errors.InvalidRecordError(str(e))
62
class ContainerSerialiser(object):
63
"""A helper class for serialising containers.
65
It simply returns bytes from method calls to 'begin', 'end' and
66
'bytes_record'. You may find ContainerWriter to be a more convenient
71
"""Return the bytes to begin a container."""
72
return FORMAT_ONE + "\n"
75
"""Return the bytes to finish a container."""
78
def bytes_record(self, bytes, names):
79
"""Return the bytes for a Bytes record with the given name and
85
byte_sections.append(str(len(bytes)) + "\n")
87
for name_tuple in names:
88
# Make sure we're writing valid names. Note that we will leave a
89
# half-written record if a name is bad!
90
for name in name_tuple:
92
byte_sections.append('\x00'.join(name_tuple) + "\n")
94
byte_sections.append("\n")
95
# Finally, the contents.
96
byte_sections.append(bytes)
97
# XXX: This causes a memory copy of bytes in size, but is usually
98
# faster than two write calls (12 vs 13 seconds to output a gig of
99
# 1k records.) - results may differ on significantly larger records
100
# like .iso's but as they should be rare in any case and thus not
101
# likely to be the common case. The biggest issue is causing extreme
102
# memory pressure in that case. One possibly improvement here is to
103
# check the size of the content before deciding to join here vs call
105
return ''.join(byte_sections)
108
class ContainerWriter(object):
109
"""A class for writing containers to a file.
111
:attribute records_written: The number of user records added to the
112
container. This does not count the prelude or suffix of the container
113
introduced by the begin() and end() methods.
116
def __init__(self, write_func):
119
:param write_func: a callable that will be called when this
120
ContainerWriter needs to write some bytes.
122
self._write_func = write_func
123
self.current_offset = 0
124
self.records_written = 0
125
self._serialiser = ContainerSerialiser()
128
"""Begin writing a container."""
129
self.write_func(self._serialiser.begin())
131
def write_func(self, bytes):
132
self._write_func(bytes)
133
self.current_offset += len(bytes)
136
"""Finish writing a container."""
137
self.write_func(self._serialiser.end())
139
def add_bytes_record(self, bytes, names):
140
"""Add a Bytes record with the given names.
142
:param bytes: The bytes to insert.
143
:param names: The names to give the inserted bytes. Each name is
144
a tuple of bytestrings. The bytestrings may not contain
146
:return: An offset, length tuple. The offset is the offset
147
of the record within the container, and the length is the
148
length of data that will need to be read to reconstitute the
149
record. These offset and length can only be used with the pack
150
interface - they might be offset by headers or other such details
151
and thus are only suitable for use by a ContainerReader.
153
current_offset = self.current_offset
154
serialised_record = self._serialiser.bytes_record(bytes, names)
155
self.write_func(serialised_record)
156
self.records_written += 1
157
# return a memo of where we wrote data to allow random access.
158
return current_offset, self.current_offset - current_offset
161
class ReadVFile(object):
162
"""Adapt a readv result iterator to a file like protocol.
164
The readv result must support the iterator protocol returning (offset,
168
# XXX: This could be a generic transport class, as other code may want to
169
# gradually consume the readv result.
171
def __init__(self, readv_result):
172
"""Construct a new ReadVFile wrapper.
174
:seealso: make_readv_reader
176
:param readv_result: the most recent readv result - list or generator
178
# we rely on its state as a generator to keep track of how much has
180
if not getattr(readv_result, 'next'):
181
readv_result = iter(readv_result)
182
self.readv_result = readv_result
186
if (self._string is None or
187
self._string.tell() == self._string_length):
188
offset, data = self.readv_result.next()
189
self._string_length = len(data)
190
self._string = StringIO(data)
192
def read(self, length):
194
result = self._string.read(length)
195
if len(result) < length:
196
raise errors.BzrError('wanted %d bytes but next '
197
'hunk only contains %d: %r...' %
198
(length, len(result), result[:20]))
202
"""Note that readline will not cross readv segments."""
204
result = self._string.readline()
205
if self._string.tell() == self._string_length and result[-1] != '\n':
206
raise errors.BzrError('short readline in the readvfile hunk: %r'
211
def make_readv_reader(transport, filename, requested_records):
212
"""Create a ContainerReader that will read selected records only.
214
:param transport: The transport the pack file is located on.
215
:param filename: The filename of the pack file.
216
:param requested_records: The record offset, length tuples as returned
217
by add_bytes_record for the desired records.
219
readv_blocks = [(0, len(FORMAT_ONE)+1)]
220
readv_blocks.extend(requested_records)
221
result = ContainerReader(ReadVFile(
222
transport.readv(filename, readv_blocks)))
226
class BaseReader(object):
228
def __init__(self, source_file):
231
:param source_file: a file-like object with `read` and `readline`
234
self._source = source_file
236
def reader_func(self, length=None):
237
return self._source.read(length)
239
def _read_line(self):
240
line = self._source.readline()
241
if not line.endswith('\n'):
242
raise errors.UnexpectedEndOfContainerError()
243
return line.rstrip('\n')
246
class ContainerReader(BaseReader):
247
"""A class for reading Bazaar's container format."""
249
def iter_records(self):
250
"""Iterate over the container, yielding each record as it is read.
252
Each yielded record will be a 2-tuple of (names, callable), where names
253
is a ``list`` and bytes is a function that takes one argument,
256
You **must not** call the callable after advancing the iterator to the
257
next record. That is, this code is invalid::
259
record_iter = container.iter_records()
260
names1, callable1 = record_iter.next()
261
names2, callable2 = record_iter.next()
262
bytes1 = callable1(None)
264
As it will give incorrect results and invalidate the state of the
267
:raises ContainerError: if any sort of container corruption is
268
detected, e.g. UnknownContainerFormatError is the format of the
269
container is unrecognised.
270
:seealso: ContainerReader.read
273
return self._iter_records()
275
def iter_record_objects(self):
276
"""Iterate over the container, yielding each record as it is read.
278
Each yielded record will be an object with ``read`` and ``validate``
279
methods. Like with iter_records, it is not safe to use a record object
280
after advancing the iterator to yield next record.
282
:raises ContainerError: if any sort of container corruption is
283
detected, e.g. UnknownContainerFormatError is the format of the
284
container is unrecognised.
285
:seealso: iter_records
288
return self._iter_record_objects()
290
def _iter_records(self):
291
for record in self._iter_record_objects():
294
def _iter_record_objects(self):
296
record_kind = self.reader_func(1)
297
if record_kind == 'B':
299
reader = BytesRecordReader(self._source)
301
elif record_kind == 'E':
302
# End marker. There are no more records.
304
elif record_kind == '':
305
# End of stream encountered, but no End Marker record seen, so
306
# this container is incomplete.
307
raise errors.UnexpectedEndOfContainerError()
309
# Unknown record type.
310
raise errors.UnknownRecordTypeError(record_kind)
312
def _read_format(self):
313
format = self._read_line()
314
if format != FORMAT_ONE:
315
raise errors.UnknownContainerFormatError(format)
318
"""Validate this container and its records.
320
Validating consumes the data stream just like iter_records and
321
iter_record_objects, so you cannot call it after
322
iter_records/iter_record_objects.
324
:raises ContainerError: if something is invalid.
327
for record_names, read_bytes in self.iter_records():
329
for name_tuple in record_names:
330
for name in name_tuple:
331
_check_name_encoding(name)
332
# Check that the name is unique. Note that Python will refuse
333
# to decode non-shortest forms of UTF-8 encoding, so there is no
334
# risk that the same unicode string has been encoded two
336
if name_tuple in all_names:
337
raise errors.DuplicateRecordNameError(name_tuple)
338
all_names.add(name_tuple)
339
excess_bytes = self.reader_func(1)
340
if excess_bytes != '':
341
raise errors.ContainerHasExcessDataError(excess_bytes)
344
class BytesRecordReader(BaseReader):
349
You can either validate or read a record, you can't do both.
351
:returns: A tuple of (names, callable). The callable can be called
352
repeatedly to obtain the bytes for the record, with a max_length
353
argument. If max_length is None, returns all the bytes. Because
354
records can be arbitrarily large, using None is not recommended
355
unless you have reason to believe the content will fit in memory.
357
# Read the content length.
358
length_line = self._read_line()
360
length = int(length_line)
362
raise errors.InvalidRecordError(
363
"%r is not a valid length." % (length_line,))
365
# Read the list of names.
368
name_line = self._read_line()
371
name_tuple = tuple(name_line.split('\x00'))
372
for name in name_tuple:
374
names.append(name_tuple)
376
self._remaining_length = length
377
return names, self._content_reader
379
def _content_reader(self, max_length):
380
if max_length is None:
381
length_to_read = self._remaining_length
383
length_to_read = min(max_length, self._remaining_length)
384
self._remaining_length -= length_to_read
385
bytes = self.reader_func(length_to_read)
386
if len(bytes) != length_to_read:
387
raise errors.UnexpectedEndOfContainerError()
391
"""Validate this record.
393
You can either validate or read, you can't do both.
395
:raises ContainerError: if this record is invalid.
397
names, read_bytes = self.read()
398
for name_tuple in names:
399
for name in name_tuple:
400
_check_name_encoding(name)
404
class ContainerPushParser(object):
405
"""A "push" parser for container format 1.
407
It accepts bytes via the ``accept_bytes`` method, and parses them into
408
records which can be retrieved via the ``read_pending_records`` method.
413
self._state_handler = self._state_expecting_format_line
414
self._parsed_records = []
415
self._reset_current_record()
416
self.finished = False
418
def _reset_current_record(self):
419
self._current_record_length = None
420
self._current_record_names = []
422
def accept_bytes(self, bytes):
423
self._buffer += bytes
424
# Keep iterating the state machine until it stops consuming bytes from
426
last_buffer_length = None
427
cur_buffer_length = len(self._buffer)
428
last_state_handler = None
429
while (cur_buffer_length != last_buffer_length
430
or last_state_handler != self._state_handler):
431
last_buffer_length = cur_buffer_length
432
last_state_handler = self._state_handler
433
self._state_handler()
434
cur_buffer_length = len(self._buffer)
436
def read_pending_records(self, max=None):
438
records = self._parsed_records[:max]
439
del self._parsed_records[:max]
442
records = self._parsed_records
443
self._parsed_records = []
446
def _consume_line(self):
447
"""Take a line out of the buffer, and return the line.
449
If a newline byte is not found in the buffer, the buffer is
450
unchanged and this returns None instead.
452
newline_pos = self._buffer.find('\n')
453
if newline_pos != -1:
454
line = self._buffer[:newline_pos]
455
self._buffer = self._buffer[newline_pos+1:]
460
def _state_expecting_format_line(self):
461
line = self._consume_line()
463
if line != FORMAT_ONE:
464
raise errors.UnknownContainerFormatError(line)
465
self._state_handler = self._state_expecting_record_type
467
def _state_expecting_record_type(self):
468
if len(self._buffer) >= 1:
469
record_type = self._buffer[0]
470
self._buffer = self._buffer[1:]
471
if record_type == 'B':
472
self._state_handler = self._state_expecting_length
473
elif record_type == 'E':
475
self._state_handler = self._state_expecting_nothing
477
raise errors.UnknownRecordTypeError(record_type)
479
def _state_expecting_length(self):
480
line = self._consume_line()
483
self._current_record_length = int(line)
485
raise errors.InvalidRecordError(
486
"%r is not a valid length." % (line,))
487
self._state_handler = self._state_expecting_name
489
def _state_expecting_name(self):
490
encoded_name_parts = self._consume_line()
491
if encoded_name_parts == '':
492
self._state_handler = self._state_expecting_body
493
elif encoded_name_parts:
494
name_parts = tuple(encoded_name_parts.split('\x00'))
495
for name_part in name_parts:
496
_check_name(name_part)
497
self._current_record_names.append(name_parts)
499
def _state_expecting_body(self):
500
if len(self._buffer) >= self._current_record_length:
501
body_bytes = self._buffer[:self._current_record_length]
502
self._buffer = self._buffer[self._current_record_length:]
503
record = (self._current_record_names, body_bytes)
504
self._parsed_records.append(record)
505
self._reset_current_record()
506
self._state_handler = self._state_expecting_record_type
508
def _state_expecting_nothing(self):
511
def read_size_hint(self):
513
if self._state_handler == self._state_expecting_body:
514
remaining = self._current_record_length - len(self._buffer)
517
return max(hint, remaining)
521
def iter_records_from_file(source_file):
522
parser = ContainerPushParser()
524
bytes = source_file.read(parser.read_size_hint())
525
parser.accept_bytes(bytes)
526
for record in parser.read_pending_records():
added
removed
bzrlib/pack.py
