1
# Copyright (C) 2007, 2009, 2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Container format for Bazaar data.
19
"Containers" and "records" are described in
20
doc/developers/container-format.txt.
23
from __future__ import absolute_import
28
from ..sixish import (
33
FORMAT_ONE = b"Bazaar pack format 1 (introduced in 0.18)"
36
_whitespace_re = re.compile(b'[\t\n\x0b\x0c\r ]')
39
def _check_name(name):
40
"""Do some basic checking of 'name'.
42
At the moment, this just checks that there are no whitespace characters in a
45
:raises InvalidRecordError: if name is not valid.
46
:seealso: _check_name_encoding
48
if _whitespace_re.search(name) is not None:
49
raise errors.InvalidRecordError("%r is not a valid name." % (name,))
52
def _check_name_encoding(name):
53
"""Check that 'name' is valid UTF-8.
55
This is separate from _check_name because UTF-8 decoding is relatively
56
expensive, and we usually want to avoid it.
58
:raises InvalidRecordError: if name is not valid UTF-8.
62
except UnicodeDecodeError as e:
63
raise errors.InvalidRecordError(str(e))
66
class ContainerSerialiser(object):
67
"""A helper class for serialising containers.
69
It simply returns bytes from method calls to 'begin', 'end' and
70
'bytes_record'. You may find ContainerWriter to be a more convenient
75
"""Return the bytes to begin a container."""
76
return FORMAT_ONE + b"\n"
79
"""Return the bytes to finish a container."""
82
def bytes_header(self, length, names):
83
"""Return the header for a Bytes record."""
85
byte_sections = [b"B"]
87
byte_sections.append(b"%d\n" % (length,))
89
for name_tuple in names:
90
# Make sure we're writing valid names. Note that we will leave a
91
# half-written record if a name is bad!
92
for name in name_tuple:
94
byte_sections.append(b'\x00'.join(name_tuple) + b"\n")
96
byte_sections.append(b"\n")
97
return b''.join(byte_sections)
99
def bytes_record(self, bytes, names):
100
"""Return the bytes for a Bytes record with the given name and
103
If the content may be large, construct the header separately and then
104
stream out the contents.
106
return self.bytes_header(len(bytes), names) + bytes
109
class ContainerWriter(object):
110
"""A class for writing containers to a file.
112
:attribute records_written: The number of user records added to the
113
container. This does not count the prelude or suffix of the container
114
introduced by the begin() and end() methods.
117
# Join up headers with the body if writing fewer than this many bytes:
118
# trades off memory usage and copying to do less IO ops.
119
_JOIN_WRITES_THRESHOLD = 100000
121
def __init__(self, write_func):
124
:param write_func: a callable that will be called when this
125
ContainerWriter needs to write some bytes.
127
self._write_func = write_func
128
self.current_offset = 0
129
self.records_written = 0
130
self._serialiser = ContainerSerialiser()
133
"""Begin writing a container."""
134
self.write_func(self._serialiser.begin())
136
def write_func(self, bytes):
137
self._write_func(bytes)
138
self.current_offset += len(bytes)
141
"""Finish writing a container."""
142
self.write_func(self._serialiser.end())
144
def add_bytes_record(self, bytes, names):
145
"""Add a Bytes record with the given names.
147
:param bytes: The bytes to insert.
148
:param names: The names to give the inserted bytes. Each name is
149
a tuple of bytestrings. The bytestrings may not contain
151
:return: An offset, length tuple. The offset is the offset
152
of the record within the container, and the length is the
153
length of data that will need to be read to reconstitute the
154
record. These offset and length can only be used with the pack
155
interface - they might be offset by headers or other such details
156
and thus are only suitable for use by a ContainerReader.
158
current_offset = self.current_offset
160
if length < self._JOIN_WRITES_THRESHOLD:
161
self.write_func(self._serialiser.bytes_header(length, names)
164
self.write_func(self._serialiser.bytes_header(length, names))
165
self.write_func(bytes)
166
self.records_written += 1
167
# return a memo of where we wrote data to allow random access.
168
return current_offset, self.current_offset - current_offset
171
class ReadVFile(object):
172
"""Adapt a readv result iterator to a file like protocol.
174
The readv result must support the iterator protocol returning (offset,
178
# XXX: This could be a generic transport class, as other code may want to
179
# gradually consume the readv result.
181
def __init__(self, readv_result):
182
"""Construct a new ReadVFile wrapper.
184
:seealso: make_readv_reader
186
:param readv_result: the most recent readv result - list or generator
188
# readv can return a sequence or an iterator, but we require an
189
# iterator to know how much has been consumed.
190
readv_result = iter(readv_result)
191
self.readv_result = readv_result
195
if (self._string is None or
196
self._string.tell() == self._string_length):
197
offset, data = next(self.readv_result)
198
self._string_length = len(data)
199
self._string = BytesIO(data)
201
def read(self, length):
203
result = self._string.read(length)
204
if len(result) < length:
205
raise errors.BzrError('wanted %d bytes but next '
206
'hunk only contains %d: %r...' %
207
(length, len(result), result[:20]))
211
"""Note that readline will not cross readv segments."""
213
result = self._string.readline()
214
if self._string.tell() == self._string_length and result[-1:] != b'\n':
215
raise errors.BzrError('short readline in the readvfile hunk: %r'
220
def make_readv_reader(transport, filename, requested_records):
221
"""Create a ContainerReader that will read selected records only.
223
:param transport: The transport the pack file is located on.
224
:param filename: The filename of the pack file.
225
:param requested_records: The record offset, length tuples as returned
226
by add_bytes_record for the desired records.
228
readv_blocks = [(0, len(FORMAT_ONE) + 1)]
229
readv_blocks.extend(requested_records)
230
result = ContainerReader(ReadVFile(
231
transport.readv(filename, readv_blocks)))
235
class BaseReader(object):
237
def __init__(self, source_file):
240
:param source_file: a file-like object with `read` and `readline`
243
self._source = source_file
245
def reader_func(self, length=None):
246
return self._source.read(length)
248
def _read_line(self):
249
line = self._source.readline()
250
if not line.endswith(b'\n'):
251
raise errors.UnexpectedEndOfContainerError()
252
return line.rstrip(b'\n')
255
class ContainerReader(BaseReader):
256
"""A class for reading Bazaar's container format."""
258
def iter_records(self):
259
"""Iterate over the container, yielding each record as it is read.
261
Each yielded record will be a 2-tuple of (names, callable), where names
262
is a ``list`` and bytes is a function that takes one argument,
265
You **must not** call the callable after advancing the iterator to the
266
next record. That is, this code is invalid::
268
record_iter = container.iter_records()
269
names1, callable1 = record_iter.next()
270
names2, callable2 = record_iter.next()
271
bytes1 = callable1(None)
273
As it will give incorrect results and invalidate the state of the
276
:raises ContainerError: if any sort of container corruption is
277
detected, e.g. UnknownContainerFormatError is the format of the
278
container is unrecognised.
279
:seealso: ContainerReader.read
282
return self._iter_records()
284
def iter_record_objects(self):
285
"""Iterate over the container, yielding each record as it is read.
287
Each yielded record will be an object with ``read`` and ``validate``
288
methods. Like with iter_records, it is not safe to use a record object
289
after advancing the iterator to yield next record.
291
:raises ContainerError: if any sort of container corruption is
292
detected, e.g. UnknownContainerFormatError is the format of the
293
container is unrecognised.
294
:seealso: iter_records
297
return self._iter_record_objects()
299
def _iter_records(self):
300
for record in self._iter_record_objects():
303
def _iter_record_objects(self):
306
record_kind = self.reader_func(1)
307
except StopIteration:
309
if record_kind == b'B':
311
reader = BytesRecordReader(self._source)
313
elif record_kind == b'E':
314
# End marker. There are no more records.
316
elif record_kind == b'':
317
# End of stream encountered, but no End Marker record seen, so
318
# this container is incomplete.
319
raise errors.UnexpectedEndOfContainerError()
321
# Unknown record type.
322
raise errors.UnknownRecordTypeError(record_kind)
324
def _read_format(self):
325
format = self._read_line()
326
if format != FORMAT_ONE:
327
raise errors.UnknownContainerFormatError(format)
330
"""Validate this container and its records.
332
Validating consumes the data stream just like iter_records and
333
iter_record_objects, so you cannot call it after
334
iter_records/iter_record_objects.
336
:raises ContainerError: if something is invalid.
339
for record_names, read_bytes in self.iter_records():
341
for name_tuple in record_names:
342
for name in name_tuple:
343
_check_name_encoding(name)
344
# Check that the name is unique. Note that Python will refuse
345
# to decode non-shortest forms of UTF-8 encoding, so there is no
346
# risk that the same unicode string has been encoded two
348
if name_tuple in all_names:
349
raise errors.DuplicateRecordNameError(name_tuple[0])
350
all_names.add(name_tuple)
351
excess_bytes = self.reader_func(1)
352
if excess_bytes != b'':
353
raise errors.ContainerHasExcessDataError(excess_bytes)
356
class BytesRecordReader(BaseReader):
361
You can either validate or read a record, you can't do both.
363
:returns: A tuple of (names, callable). The callable can be called
364
repeatedly to obtain the bytes for the record, with a max_length
365
argument. If max_length is None, returns all the bytes. Because
366
records can be arbitrarily large, using None is not recommended
367
unless you have reason to believe the content will fit in memory.
369
# Read the content length.
370
length_line = self._read_line()
372
length = int(length_line)
374
raise errors.InvalidRecordError(
375
"%r is not a valid length." % (length_line,))
377
# Read the list of names.
380
name_line = self._read_line()
383
name_tuple = tuple(name_line.split(b'\x00'))
384
for name in name_tuple:
386
names.append(name_tuple)
388
self._remaining_length = length
389
return names, self._content_reader
391
def _content_reader(self, max_length):
392
if max_length is None:
393
length_to_read = self._remaining_length
395
length_to_read = min(max_length, self._remaining_length)
396
self._remaining_length -= length_to_read
397
bytes = self.reader_func(length_to_read)
398
if len(bytes) != length_to_read:
399
raise errors.UnexpectedEndOfContainerError()
403
"""Validate this record.
405
You can either validate or read, you can't do both.
407
:raises ContainerError: if this record is invalid.
409
names, read_bytes = self.read()
410
for name_tuple in names:
411
for name in name_tuple:
412
_check_name_encoding(name)
416
class ContainerPushParser(object):
417
"""A "push" parser for container format 1.
419
It accepts bytes via the ``accept_bytes`` method, and parses them into
420
records which can be retrieved via the ``read_pending_records`` method.
425
self._state_handler = self._state_expecting_format_line
426
self._parsed_records = []
427
self._reset_current_record()
428
self.finished = False
430
def _reset_current_record(self):
431
self._current_record_length = None
432
self._current_record_names = []
434
def accept_bytes(self, bytes):
435
self._buffer += bytes
436
# Keep iterating the state machine until it stops consuming bytes from
438
last_buffer_length = None
439
cur_buffer_length = len(self._buffer)
440
last_state_handler = None
441
while (cur_buffer_length != last_buffer_length
442
or last_state_handler != self._state_handler):
443
last_buffer_length = cur_buffer_length
444
last_state_handler = self._state_handler
445
self._state_handler()
446
cur_buffer_length = len(self._buffer)
448
def read_pending_records(self, max=None):
450
records = self._parsed_records[:max]
451
del self._parsed_records[:max]
454
records = self._parsed_records
455
self._parsed_records = []
458
def _consume_line(self):
459
"""Take a line out of the buffer, and return the line.
461
If a newline byte is not found in the buffer, the buffer is
462
unchanged and this returns None instead.
464
newline_pos = self._buffer.find(b'\n')
465
if newline_pos != -1:
466
line = self._buffer[:newline_pos]
467
self._buffer = self._buffer[newline_pos + 1:]
472
def _state_expecting_format_line(self):
473
line = self._consume_line()
475
if line != FORMAT_ONE:
476
raise errors.UnknownContainerFormatError(line)
477
self._state_handler = self._state_expecting_record_type
479
def _state_expecting_record_type(self):
480
if len(self._buffer) >= 1:
481
record_type = self._buffer[:1]
482
self._buffer = self._buffer[1:]
483
if record_type == b'B':
484
self._state_handler = self._state_expecting_length
485
elif record_type == b'E':
487
self._state_handler = self._state_expecting_nothing
489
raise errors.UnknownRecordTypeError(record_type)
491
def _state_expecting_length(self):
492
line = self._consume_line()
495
self._current_record_length = int(line)
497
raise errors.InvalidRecordError(
498
"%r is not a valid length." % (line,))
499
self._state_handler = self._state_expecting_name
501
def _state_expecting_name(self):
502
encoded_name_parts = self._consume_line()
503
if encoded_name_parts == b'':
504
self._state_handler = self._state_expecting_body
505
elif encoded_name_parts:
506
name_parts = tuple(encoded_name_parts.split(b'\x00'))
507
for name_part in name_parts:
508
_check_name(name_part)
509
self._current_record_names.append(name_parts)
511
def _state_expecting_body(self):
512
if len(self._buffer) >= self._current_record_length:
513
body_bytes = self._buffer[:self._current_record_length]
514
self._buffer = self._buffer[self._current_record_length:]
515
record = (self._current_record_names, body_bytes)
516
self._parsed_records.append(record)
517
self._reset_current_record()
518
self._state_handler = self._state_expecting_record_type
520
def _state_expecting_nothing(self):
523
def read_size_hint(self):
525
if self._state_handler == self._state_expecting_body:
526
remaining = self._current_record_length - len(self._buffer)
529
return max(hint, remaining)
533
def iter_records_from_file(source_file):
534
parser = ContainerPushParser()
536
bytes = source_file.read(parser.read_size_hint())
537
parser.accept_bytes(bytes)
538
for record in parser.read_pending_records():
added
removed
breezy/bzr/pack.py
