13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
"""Container format for Bazaar data.
19
"Containers" and "records" are described in doc/developers/container-format.txt.
19
"Containers" and "records" are described in
20
doc/developers/container-format.txt.
22
23
from cStringIO import StringIO
58
59
raise errors.InvalidRecordError(str(e))
62
class ContainerSerialiser(object):
63
"""A helper class for serialising containers.
65
It simply returns bytes from method calls to 'begin', 'end' and
66
'bytes_record'. You may find ContainerWriter to be a more convenient
71
"""Return the bytes to begin a container."""
72
return FORMAT_ONE + "\n"
75
"""Return the bytes to finish a container."""
78
def bytes_record(self, bytes, names):
79
"""Return the bytes for a Bytes record with the given name and
85
byte_sections.append(str(len(bytes)) + "\n")
87
for name_tuple in names:
88
# Make sure we're writing valid names. Note that we will leave a
89
# half-written record if a name is bad!
90
for name in name_tuple:
92
byte_sections.append('\x00'.join(name_tuple) + "\n")
94
byte_sections.append("\n")
95
# Finally, the contents.
96
byte_sections.append(bytes)
97
# XXX: This causes a memory copy of bytes in size, but is usually
98
# faster than two write calls (12 vs 13 seconds to output a gig of
99
# 1k records.) - results may differ on significantly larger records
100
# like .iso's but as they should be rare in any case and thus not
101
# likely to be the common case. The biggest issue is causing extreme
102
# memory pressure in that case. One possibly improvement here is to
103
# check the size of the content before deciding to join here vs call
105
return ''.join(byte_sections)
61
108
class ContainerWriter(object):
62
"""A class for writing containers."""
109
"""A class for writing containers to a file.
111
:attribute records_written: The number of user records added to the
112
container. This does not count the prelude or suffix of the container
113
introduced by the begin() and end() methods.
64
116
def __init__(self, write_func):
97
151
and thus are only suitable for use by a ContainerReader.
99
153
current_offset = self.current_offset
103
self.write_func(str(len(bytes)) + "\n")
105
for name_tuple in names:
106
# Make sure we're writing valid names. Note that we will leave a
107
# half-written record if a name is bad!
108
for name in name_tuple:
110
self.write_func('\x00'.join(name_tuple) + "\n")
112
self.write_func("\n")
113
# Finally, the contents.
114
self.write_func(bytes)
154
serialised_record = self._serialiser.bytes_record(bytes, names)
155
self.write_func(serialised_record)
156
self.records_written += 1
115
157
# return a memo of where we wrote data to allow random access.
116
158
return current_offset, self.current_offset - current_offset
192
234
is a ``list`` and bytes is a function that takes one argument,
195
You **must not** call the callable after advancing the interator to the
237
You **must not** call the callable after advancing the iterator to the
196
238
next record. That is, this code is invalid::
198
240
record_iter = container.iter_records()
199
241
names1, callable1 = record_iter.next()
200
242
names2, callable2 = record_iter.next()
201
243
bytes1 = callable1(None)
203
245
As it will give incorrect results and invalidate the state of the
206
:raises ContainerError: if any sort of containter corruption is
248
:raises ContainerError: if any sort of container corruption is
207
249
detected, e.g. UnknownContainerFormatError is the format of the
208
250
container is unrecognised.
209
251
:seealso: ContainerReader.read
211
253
self._read_format()
212
254
return self._iter_records()
214
256
def iter_record_objects(self):
215
257
"""Iterate over the container, yielding each record as it is read.
339
381
_check_name_encoding(name)
385
class ContainerPushParser(object):
386
"""A "push" parser for container format 1.
388
It accepts bytes via the ``accept_bytes`` method, and parses them into
389
records which can be retrieved via the ``read_pending_records`` method.
394
self._state_handler = self._state_expecting_format_line
395
self._parsed_records = []
396
self._reset_current_record()
397
self.finished = False
399
def _reset_current_record(self):
400
self._current_record_length = None
401
self._current_record_names = []
403
def accept_bytes(self, bytes):
404
self._buffer += bytes
405
# Keep iterating the state machine until it stops consuming bytes from
407
last_buffer_length = None
408
cur_buffer_length = len(self._buffer)
409
while cur_buffer_length != last_buffer_length:
410
last_buffer_length = cur_buffer_length
411
self._state_handler()
412
cur_buffer_length = len(self._buffer)
414
def read_pending_records(self, max=None):
416
records = self._parsed_records[:max]
417
del self._parsed_records[:max]
420
records = self._parsed_records
421
self._parsed_records = []
424
def _consume_line(self):
425
"""Take a line out of the buffer, and return the line.
427
If a newline byte is not found in the buffer, the buffer is
428
unchanged and this returns None instead.
430
newline_pos = self._buffer.find('\n')
431
if newline_pos != -1:
432
line = self._buffer[:newline_pos]
433
self._buffer = self._buffer[newline_pos+1:]
438
def _state_expecting_format_line(self):
439
line = self._consume_line()
441
if line != FORMAT_ONE:
442
raise errors.UnknownContainerFormatError(line)
443
self._state_handler = self._state_expecting_record_type
445
def _state_expecting_record_type(self):
446
if len(self._buffer) >= 1:
447
record_type = self._buffer[0]
448
self._buffer = self._buffer[1:]
449
if record_type == 'B':
450
self._state_handler = self._state_expecting_length
451
elif record_type == 'E':
453
self._state_handler = self._state_expecting_nothing
455
raise errors.UnknownRecordTypeError(record_type)
457
def _state_expecting_length(self):
458
line = self._consume_line()
461
self._current_record_length = int(line)
463
raise errors.InvalidRecordError(
464
"%r is not a valid length." % (line,))
465
self._state_handler = self._state_expecting_name
467
def _state_expecting_name(self):
468
encoded_name_parts = self._consume_line()
469
if encoded_name_parts == '':
470
self._state_handler = self._state_expecting_body
471
elif encoded_name_parts:
472
name_parts = tuple(encoded_name_parts.split('\x00'))
473
for name_part in name_parts:
474
_check_name(name_part)
475
self._current_record_names.append(name_parts)
477
def _state_expecting_body(self):
478
if len(self._buffer) >= self._current_record_length:
479
body_bytes = self._buffer[:self._current_record_length]
480
self._buffer = self._buffer[self._current_record_length:]
481
record = (self._current_record_names, body_bytes)
482
self._parsed_records.append(record)
483
self._reset_current_record()
484
self._state_handler = self._state_expecting_record_type
486
def _state_expecting_nothing(self):
489
def read_size_hint(self):
491
if self._state_handler == self._state_expecting_body:
492
remaining = self._current_record_length - len(self._buffer)
495
return max(hint, remaining)
499
def iter_records_from_file(source_file):
500
parser = ContainerPushParser()
502
bytes = source_file.read(parser.read_size_hint())
503
parser.accept_bytes(bytes)
504
for record in parser.read_pending_records():
added
removed
bzrlib/pack.py
