1
# Copyright (C) 2007, 2009, 2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Container format for Bazaar data.
19
"Containers" and "records" are described in
20
doc/developers/container-format.txt.
23
from cStringIO import StringIO
26
from bzrlib import errors
29
FORMAT_ONE = "Bazaar pack format 1 (introduced in 0.18)"
32
_whitespace_re = re.compile('[\t\n\x0b\x0c\r ]')
35
def _check_name(name):
36
"""Do some basic checking of 'name'.
38
At the moment, this just checks that there are no whitespace characters in a
41
:raises InvalidRecordError: if name is not valid.
42
:seealso: _check_name_encoding
44
if _whitespace_re.search(name) is not None:
45
raise errors.InvalidRecordError("%r is not a valid name." % (name,))
48
def _check_name_encoding(name):
49
"""Check that 'name' is valid UTF-8.
51
This is separate from _check_name because UTF-8 decoding is relatively
52
expensive, and we usually want to avoid it.
54
:raises InvalidRecordError: if name is not valid UTF-8.
58
except UnicodeDecodeError, e:
59
raise errors.InvalidRecordError(str(e))
62
class ContainerSerialiser(object):
63
"""A helper class for serialising containers.
65
It simply returns bytes from method calls to 'begin', 'end' and
66
'bytes_record'. You may find ContainerWriter to be a more convenient
71
"""Return the bytes to begin a container."""
72
return FORMAT_ONE + "\n"
75
"""Return the bytes to finish a container."""
78
def bytes_header(self, length, names):
79
"""Return the header for a Bytes record."""
83
byte_sections.append(str(length) + "\n")
85
for name_tuple in names:
86
# Make sure we're writing valid names. Note that we will leave a
87
# half-written record if a name is bad!
88
for name in name_tuple:
90
byte_sections.append('\x00'.join(name_tuple) + "\n")
92
byte_sections.append("\n")
93
return ''.join(byte_sections)
95
def bytes_record(self, bytes, names):
96
"""Return the bytes for a Bytes record with the given name and
99
If the content may be large, construct the header separately and then
100
stream out the contents.
102
return self.bytes_header(len(bytes), names) + bytes
105
class ContainerWriter(object):
106
"""A class for writing containers to a file.
108
:attribute records_written: The number of user records added to the
109
container. This does not count the prelude or suffix of the container
110
introduced by the begin() and end() methods.
113
# Join up headers with the body if writing fewer than this many bytes:
114
# trades off memory usage and copying to do less IO ops.
115
_JOIN_WRITES_THRESHOLD = 100000
117
def __init__(self, write_func):
120
:param write_func: a callable that will be called when this
121
ContainerWriter needs to write some bytes.
123
self._write_func = write_func
124
self.current_offset = 0
125
self.records_written = 0
126
self._serialiser = ContainerSerialiser()
129
"""Begin writing a container."""
130
self.write_func(self._serialiser.begin())
132
def write_func(self, bytes):
133
self._write_func(bytes)
134
self.current_offset += len(bytes)
137
"""Finish writing a container."""
138
self.write_func(self._serialiser.end())
140
def add_bytes_record(self, bytes, names):
141
"""Add a Bytes record with the given names.
143
:param bytes: The bytes to insert.
144
:param names: The names to give the inserted bytes. Each name is
145
a tuple of bytestrings. The bytestrings may not contain
147
:return: An offset, length tuple. The offset is the offset
148
of the record within the container, and the length is the
149
length of data that will need to be read to reconstitute the
150
record. These offset and length can only be used with the pack
151
interface - they might be offset by headers or other such details
152
and thus are only suitable for use by a ContainerReader.
154
current_offset = self.current_offset
156
if length < self._JOIN_WRITES_THRESHOLD:
157
self.write_func(self._serialiser.bytes_header(length, names)
160
self.write_func(self._serialiser.bytes_header(length, names))
161
self.write_func(bytes)
162
self.records_written += 1
163
# return a memo of where we wrote data to allow random access.
164
return current_offset, self.current_offset - current_offset
167
class ReadVFile(object):
168
"""Adapt a readv result iterator to a file like protocol.
170
The readv result must support the iterator protocol returning (offset,
174
# XXX: This could be a generic transport class, as other code may want to
175
# gradually consume the readv result.
177
def __init__(self, readv_result):
178
"""Construct a new ReadVFile wrapper.
180
:seealso: make_readv_reader
182
:param readv_result: the most recent readv result - list or generator
184
# readv can return a sequence or an iterator, but we require an
185
# iterator to know how much has been consumed.
186
readv_result = iter(readv_result)
187
self.readv_result = readv_result
191
if (self._string is None or
192
self._string.tell() == self._string_length):
193
offset, data = self.readv_result.next()
194
self._string_length = len(data)
195
self._string = StringIO(data)
197
def read(self, length):
199
result = self._string.read(length)
200
if len(result) < length:
201
raise errors.BzrError('wanted %d bytes but next '
202
'hunk only contains %d: %r...' %
203
(length, len(result), result[:20]))
207
"""Note that readline will not cross readv segments."""
209
result = self._string.readline()
210
if self._string.tell() == self._string_length and result[-1] != '\n':
211
raise errors.BzrError('short readline in the readvfile hunk: %r'
216
def make_readv_reader(transport, filename, requested_records):
217
"""Create a ContainerReader that will read selected records only.
219
:param transport: The transport the pack file is located on.
220
:param filename: The filename of the pack file.
221
:param requested_records: The record offset, length tuples as returned
222
by add_bytes_record for the desired records.
224
readv_blocks = [(0, len(FORMAT_ONE)+1)]
225
readv_blocks.extend(requested_records)
226
result = ContainerReader(ReadVFile(
227
transport.readv(filename, readv_blocks)))
231
class BaseReader(object):
233
def __init__(self, source_file):
236
:param source_file: a file-like object with `read` and `readline`
239
self._source = source_file
241
def reader_func(self, length=None):
242
return self._source.read(length)
244
def _read_line(self):
245
line = self._source.readline()
246
if not line.endswith('\n'):
247
raise errors.UnexpectedEndOfContainerError()
248
return line.rstrip('\n')
251
class ContainerReader(BaseReader):
252
"""A class for reading Bazaar's container format."""
254
def iter_records(self):
255
"""Iterate over the container, yielding each record as it is read.
257
Each yielded record will be a 2-tuple of (names, callable), where names
258
is a ``list`` and bytes is a function that takes one argument,
261
You **must not** call the callable after advancing the iterator to the
262
next record. That is, this code is invalid::
264
record_iter = container.iter_records()
265
names1, callable1 = record_iter.next()
266
names2, callable2 = record_iter.next()
267
bytes1 = callable1(None)
269
As it will give incorrect results and invalidate the state of the
272
:raises ContainerError: if any sort of container corruption is
273
detected, e.g. UnknownContainerFormatError is the format of the
274
container is unrecognised.
275
:seealso: ContainerReader.read
278
return self._iter_records()
280
def iter_record_objects(self):
281
"""Iterate over the container, yielding each record as it is read.
283
Each yielded record will be an object with ``read`` and ``validate``
284
methods. Like with iter_records, it is not safe to use a record object
285
after advancing the iterator to yield next record.
287
:raises ContainerError: if any sort of container corruption is
288
detected, e.g. UnknownContainerFormatError is the format of the
289
container is unrecognised.
290
:seealso: iter_records
293
return self._iter_record_objects()
295
def _iter_records(self):
296
for record in self._iter_record_objects():
299
def _iter_record_objects(self):
301
record_kind = self.reader_func(1)
302
if record_kind == 'B':
304
reader = BytesRecordReader(self._source)
306
elif record_kind == 'E':
307
# End marker. There are no more records.
309
elif record_kind == '':
310
# End of stream encountered, but no End Marker record seen, so
311
# this container is incomplete.
312
raise errors.UnexpectedEndOfContainerError()
314
# Unknown record type.
315
raise errors.UnknownRecordTypeError(record_kind)
317
def _read_format(self):
318
format = self._read_line()
319
if format != FORMAT_ONE:
320
raise errors.UnknownContainerFormatError(format)
323
"""Validate this container and its records.
325
Validating consumes the data stream just like iter_records and
326
iter_record_objects, so you cannot call it after
327
iter_records/iter_record_objects.
329
:raises ContainerError: if something is invalid.
332
for record_names, read_bytes in self.iter_records():
334
for name_tuple in record_names:
335
for name in name_tuple:
336
_check_name_encoding(name)
337
# Check that the name is unique. Note that Python will refuse
338
# to decode non-shortest forms of UTF-8 encoding, so there is no
339
# risk that the same unicode string has been encoded two
341
if name_tuple in all_names:
342
raise errors.DuplicateRecordNameError(name_tuple[0])
343
all_names.add(name_tuple)
344
excess_bytes = self.reader_func(1)
345
if excess_bytes != '':
346
raise errors.ContainerHasExcessDataError(excess_bytes)
349
class BytesRecordReader(BaseReader):
354
You can either validate or read a record, you can't do both.
356
:returns: A tuple of (names, callable). The callable can be called
357
repeatedly to obtain the bytes for the record, with a max_length
358
argument. If max_length is None, returns all the bytes. Because
359
records can be arbitrarily large, using None is not recommended
360
unless you have reason to believe the content will fit in memory.
362
# Read the content length.
363
length_line = self._read_line()
365
length = int(length_line)
367
raise errors.InvalidRecordError(
368
"%r is not a valid length." % (length_line,))
370
# Read the list of names.
373
name_line = self._read_line()
376
name_tuple = tuple(name_line.split('\x00'))
377
for name in name_tuple:
379
names.append(name_tuple)
381
self._remaining_length = length
382
return names, self._content_reader
384
def _content_reader(self, max_length):
385
if max_length is None:
386
length_to_read = self._remaining_length
388
length_to_read = min(max_length, self._remaining_length)
389
self._remaining_length -= length_to_read
390
bytes = self.reader_func(length_to_read)
391
if len(bytes) != length_to_read:
392
raise errors.UnexpectedEndOfContainerError()
396
"""Validate this record.
398
You can either validate or read, you can't do both.
400
:raises ContainerError: if this record is invalid.
402
names, read_bytes = self.read()
403
for name_tuple in names:
404
for name in name_tuple:
405
_check_name_encoding(name)
409
class ContainerPushParser(object):
410
"""A "push" parser for container format 1.
412
It accepts bytes via the ``accept_bytes`` method, and parses them into
413
records which can be retrieved via the ``read_pending_records`` method.
418
self._state_handler = self._state_expecting_format_line
419
self._parsed_records = []
420
self._reset_current_record()
421
self.finished = False
423
def _reset_current_record(self):
424
self._current_record_length = None
425
self._current_record_names = []
427
def accept_bytes(self, bytes):
428
self._buffer += bytes
429
# Keep iterating the state machine until it stops consuming bytes from
431
last_buffer_length = None
432
cur_buffer_length = len(self._buffer)
433
last_state_handler = None
434
while (cur_buffer_length != last_buffer_length
435
or last_state_handler != self._state_handler):
436
last_buffer_length = cur_buffer_length
437
last_state_handler = self._state_handler
438
self._state_handler()
439
cur_buffer_length = len(self._buffer)
441
def read_pending_records(self, max=None):
443
records = self._parsed_records[:max]
444
del self._parsed_records[:max]
447
records = self._parsed_records
448
self._parsed_records = []
451
def _consume_line(self):
452
"""Take a line out of the buffer, and return the line.
454
If a newline byte is not found in the buffer, the buffer is
455
unchanged and this returns None instead.
457
newline_pos = self._buffer.find('\n')
458
if newline_pos != -1:
459
line = self._buffer[:newline_pos]
460
self._buffer = self._buffer[newline_pos+1:]
465
def _state_expecting_format_line(self):
466
line = self._consume_line()
468
if line != FORMAT_ONE:
469
raise errors.UnknownContainerFormatError(line)
470
self._state_handler = self._state_expecting_record_type
472
def _state_expecting_record_type(self):
473
if len(self._buffer) >= 1:
474
record_type = self._buffer[0]
475
self._buffer = self._buffer[1:]
476
if record_type == 'B':
477
self._state_handler = self._state_expecting_length
478
elif record_type == 'E':
480
self._state_handler = self._state_expecting_nothing
482
raise errors.UnknownRecordTypeError(record_type)
484
def _state_expecting_length(self):
485
line = self._consume_line()
488
self._current_record_length = int(line)
490
raise errors.InvalidRecordError(
491
"%r is not a valid length." % (line,))
492
self._state_handler = self._state_expecting_name
494
def _state_expecting_name(self):
495
encoded_name_parts = self._consume_line()
496
if encoded_name_parts == '':
497
self._state_handler = self._state_expecting_body
498
elif encoded_name_parts:
499
name_parts = tuple(encoded_name_parts.split('\x00'))
500
for name_part in name_parts:
501
_check_name(name_part)
502
self._current_record_names.append(name_parts)
504
def _state_expecting_body(self):
505
if len(self._buffer) >= self._current_record_length:
506
body_bytes = self._buffer[:self._current_record_length]
507
self._buffer = self._buffer[self._current_record_length:]
508
record = (self._current_record_names, body_bytes)
509
self._parsed_records.append(record)
510
self._reset_current_record()
511
self._state_handler = self._state_expecting_record_type
513
def _state_expecting_nothing(self):
516
def read_size_hint(self):
518
if self._state_handler == self._state_expecting_body:
519
remaining = self._current_record_length - len(self._buffer)
522
return max(hint, remaining)
526
def iter_records_from_file(source_file):
527
parser = ContainerPushParser()
529
bytes = source_file.read(parser.read_size_hint())
530
parser.accept_bytes(bytes)
531
for record in parser.read_pending_records():
added
removed
bzrlib/pack.py
