64
74
return "&#%d;" % ord(match.group())
67
_unicode_to_escaped_map = {}
69
def _encode_and_escape(unicode_str, _map=_unicode_to_escaped_map):
77
def _utf8_escape_replace(match, _map=_xml_escape_map):
78
"""Escape utf8 characters into XML safe ones.
80
This uses 2 tricks. It is either escaping "standard" characters, like "&<>,
81
or it is handling characters with the high-bit set. For ascii characters,
82
we just lookup the replacement in the dictionary. For everything else, we
83
decode back into Unicode, and then use the XML escape code.
86
return _map[match.group()]
88
return ''.join('&#%d;' % ord(uni_chr)
89
for uni_chr in match.group().decode('utf8'))
94
def _encode_and_escape(unicode_or_utf8_str, _map=_to_escaped_map):
70
95
"""Encode the string into utf8, and escape invalid XML characters"""
71
96
# We frequently get entities we have not seen before, so it is better
72
97
# to check if None, rather than try/KeyError
73
text = _map.get(unicode_str)
98
text = _map.get(unicode_or_utf8_str)
75
# The alternative policy is to do a regular UTF8 encoding
76
# and then escape only XML meta characters.
77
# Performance is equivalent once you use cache_utf8. *However*
78
# this makes the serialized texts incompatible with old versions
79
# of bzr. So no net gain. (Perhaps the read code would handle utf8
80
# better than entity escapes, but cElementTree seems to do just fine
82
text = str(_utf8_re.sub(_utf8_escape_replace, unicode_str)) + '"'
83
_map[unicode_str] = text
100
if unicode_or_utf8_str.__class__ is unicode:
101
# The alternative policy is to do a regular UTF8 encoding
102
# and then escape only XML meta characters.
103
# Performance is equivalent once you use cache_utf8. *However*
104
# this makes the serialized texts incompatible with old versions
105
# of bzr. So no net gain. (Perhaps the read code would handle utf8
106
# better than entity escapes, but cElementTree seems to do just fine
108
text = str(_unicode_re.sub(_unicode_escape_replace,
109
unicode_or_utf8_str)) + '"'
111
# Plain strings are considered to already be in utf-8 so we do a
112
# slightly different method for escaping.
113
text = _utf8_re.sub(_utf8_escape_replace,
114
unicode_or_utf8_str) + '"'
115
_map[unicode_or_utf8_str] = text
119
def _get_utf8_or_ascii(a_str,
120
_encode_utf8=cache_utf8.encode,
121
_get_cached_ascii=cache_utf8.get_cached_ascii):
122
"""Return a cached version of the string.
124
cElementTree will return a plain string if the XML is plain ascii. It only
125
returns Unicode when it needs to. We want to work in utf-8 strings. So if
126
cElementTree returns a plain string, we can just return the cached version.
127
If it is Unicode, then we need to encode it.
129
:param a_str: An 8-bit string or Unicode as returned by
130
cElementTree.Element.get()
131
:return: A utf-8 encoded 8-bit string.
133
# This is fairly optimized because we know what cElementTree does, this is
134
# not meant as a generic function for all cases. Because it is possible for
135
# an 8-bit string to not be ascii or valid utf8.
136
if a_str.__class__ is unicode:
137
return _encode_utf8(a_str)
87
142
def _clear_cache():
88
143
"""Clean out the unicode => escaped map"""
89
_unicode_to_escaped_map.clear()
92
class Serializer_v5(Serializer):
93
"""Version 5 serializer
95
Packs objects into XML and vice versa.
144
_to_escaped_map.clear()
147
class Serializer_v8(XMLSerializer):
148
"""This serialiser adds rich roots.
150
Its revision format number matches its inventory number.
100
156
support_altered_by_hack = True
101
157
# This format supports the altered-by hack that reads file ids directly out
102
158
# of the versionedfile, without doing XML parsing.
104
def write_inventory_to_string(self, inv):
105
"""Just call write_inventory with a StringIO and return the value"""
160
supported_kinds = set(['file', 'directory', 'symlink'])
162
revision_format_num = None
164
def _check_revisions(self, inv):
165
"""Extension point for subclasses to check during serialisation.
167
:param inv: An inventory about to be serialised, to be checked.
168
:raises: AssertionError if an error has occurred.
170
if inv.revision_id is None:
171
raise AssertionError("inv.revision_id is None")
172
if inv.root.revision is None:
173
raise AssertionError("inv.root.revision is None")
175
def _check_cache_size(self, inv_size, entry_cache):
176
"""Check that the entry_cache is large enough.
178
We want the cache to be ~2x the size of an inventory. The reason is
179
because we use a FIFO cache, and how Inventory records are likely to
180
change. In general, you have a small number of records which change
181
often, and a lot of records which do not change at all. So when the
182
cache gets full, you actually flush out a lot of the records you are
183
interested in, which means you need to recreate all of those records.
184
An LRU Cache would be better, but the overhead negates the cache
187
One way to look at it, only the size of the cache > len(inv) is your
188
'working' set. And in general, it shouldn't be a problem to hold 2
189
inventories in memory anyway.
191
:param inv_size: The number of entries in an inventory.
193
if entry_cache is None:
195
# 1.5 times might also be reasonable.
196
recommended_min_cache_size = inv_size * 1.5
197
if entry_cache.cache_size() < recommended_min_cache_size:
198
recommended_cache_size = inv_size * 2
199
trace.mutter('Resizing the inventory entry cache from %d to %d',
200
entry_cache.cache_size(), recommended_cache_size)
201
entry_cache.resize(recommended_cache_size)
203
def write_inventory_to_lines(self, inv):
204
"""Return a list of lines with the encoded inventory."""
205
return self.write_inventory(inv, None)
207
def write_inventory_to_string(self, inv, working=False):
208
"""Just call write_inventory with a StringIO and return the value.
210
:param working: If True skip history data - text_sha1, text_size,
211
reference_revision, symlink_target.
106
213
sio = cStringIO.StringIO()
107
self.write_inventory(inv, sio)
214
self.write_inventory(inv, sio, working)
108
215
return sio.getvalue()
110
def write_inventory(self, inv, f):
217
def write_inventory(self, inv, f, working=False):
111
218
"""Write inventory to a file.
113
220
:param inv: the inventory to write.
114
:param f: the file to write.
221
:param f: the file to write. (May be None if the lines are the desired
223
:param working: If True skip history data - text_sha1, text_size,
224
reference_revision, symlink_target.
225
:return: The inventory as a list of lines.
116
227
_ensure_utf8_re()
228
self._check_revisions(inv)
118
230
append = output.append
119
231
self._append_inventory_root(append, inv)
122
234
root_path, root_ie = entries.next()
123
235
for path, ie in entries:
124
self._append_entry(append, ie)
236
if ie.parent_id != self.root_id:
237
parent_str = ' parent_id="'
238
parent_id = _encode_and_escape(ie.parent_id)
242
if ie.kind == 'file':
244
executable = ' executable="yes"'
248
append('<file%s file_id="%s name="%s%s%s revision="%s '
249
'text_sha1="%s" text_size="%d" />\n' % (
250
executable, _encode_and_escape(ie.file_id),
251
_encode_and_escape(ie.name), parent_str, parent_id,
252
_encode_and_escape(ie.revision), ie.text_sha1,
255
append('<file%s file_id="%s name="%s%s%s />\n' % (
256
executable, _encode_and_escape(ie.file_id),
257
_encode_and_escape(ie.name), parent_str, parent_id))
258
elif ie.kind == 'directory':
260
append('<directory file_id="%s name="%s%s%s revision="%s '
262
_encode_and_escape(ie.file_id),
263
_encode_and_escape(ie.name),
264
parent_str, parent_id,
265
_encode_and_escape(ie.revision)))
267
append('<directory file_id="%s name="%s%s%s />\n' % (
268
_encode_and_escape(ie.file_id),
269
_encode_and_escape(ie.name),
270
parent_str, parent_id))
271
elif ie.kind == 'symlink':
273
append('<symlink file_id="%s name="%s%s%s revision="%s '
274
'symlink_target="%s />\n' % (
275
_encode_and_escape(ie.file_id),
276
_encode_and_escape(ie.name),
277
parent_str, parent_id,
278
_encode_and_escape(ie.revision),
279
_encode_and_escape(ie.symlink_target)))
281
append('<symlink file_id="%s name="%s%s%s />\n' % (
282
_encode_and_escape(ie.file_id),
283
_encode_and_escape(ie.name),
284
parent_str, parent_id))
285
elif ie.kind == 'tree-reference':
286
if ie.kind not in self.supported_kinds:
287
raise errors.UnsupportedInventoryKind(ie.kind)
289
append('<tree-reference file_id="%s name="%s%s%s '
290
'revision="%s reference_revision="%s />\n' % (
291
_encode_and_escape(ie.file_id),
292
_encode_and_escape(ie.name),
293
parent_str, parent_id,
294
_encode_and_escape(ie.revision),
295
_encode_and_escape(ie.reference_revision)))
297
append('<tree-reference file_id="%s name="%s%s%s />\n' % (
298
_encode_and_escape(ie.file_id),
299
_encode_and_escape(ie.name),
300
parent_str, parent_id))
302
raise errors.UnsupportedInventoryKind(ie.kind)
125
303
append('</inventory>\n')
127
306
# Just to keep the cache from growing without bounds
128
307
# but we may actually not want to do clear the cache
131
311
def _append_inventory_root(self, append, inv):
132
312
"""Append the inventory root to output."""
134
if inv.root.file_id not in (None, ROOT_ID):
136
append(_encode_and_escape(inv.root.file_id))
137
append(' format="5"')
138
313
if inv.revision_id is not None:
139
append(' revision_id="')
140
append(_encode_and_escape(inv.revision_id))
143
def _append_entry(self, append, ie):
144
"""Convert InventoryEntry to XML element and append to output."""
145
# TODO: should just be a plain assertion
146
assert InventoryEntry.versionable_kind(ie.kind), \
147
'unsupported entry kind %s' % ie.kind
152
append(' executable="yes"')
154
append(_encode_and_escape(ie.file_id))
156
append(_encode_and_escape(ie.name))
157
if self._parent_condition(ie):
158
assert isinstance(ie.parent_id, basestring)
159
append(' parent_id="')
160
append(_encode_and_escape(ie.parent_id))
161
if ie.revision is not None:
162
append(' revision="')
163
append(_encode_and_escape(ie.revision))
164
if ie.symlink_target is not None:
165
append(' symlink_target="')
166
append(_encode_and_escape(ie.symlink_target))
167
if ie.text_sha1 is not None:
168
append(' text_sha1="')
171
if ie.text_size is not None:
172
append(' text_size="%d"' % ie.text_size)
176
def _parent_condition(self, ie):
177
return ie.parent_id != ROOT_ID
314
revid1 = ' revision_id="'
315
revid2 = _encode_and_escape(inv.revision_id)
319
append('<inventory format="%s"%s%s>\n' % (
320
self.format_num, revid1, revid2))
321
append('<directory file_id="%s name="%s revision="%s />\n' % (
322
_encode_and_escape(inv.root.file_id),
323
_encode_and_escape(inv.root.name),
324
_encode_and_escape(inv.root.revision)))
179
326
def _pack_revision(self, rev):
180
327
"""Revision object -> xml tree"""
328
# For the XML format, we need to write them as Unicode rather than as
329
# utf-8 strings. So that cElementTree can handle properly escaping
331
decode_utf8 = cache_utf8.decode
332
revision_id = rev.revision_id
333
if isinstance(revision_id, str):
334
revision_id = decode_utf8(revision_id)
335
format_num = self.format_num
336
if self.revision_format_num is not None:
337
format_num = self.revision_format_num
181
338
root = Element('revision',
182
339
committer = rev.committer,
183
timestamp = '%.9f' % rev.timestamp,
184
revision_id = rev.revision_id,
340
timestamp = '%.3f' % rev.timestamp,
341
revision_id = revision_id,
185
342
inventory_sha1 = rev.inventory_sha1,
188
345
if rev.timezone is not None:
189
346
root.set('timezone', str(rev.timezone))
191
348
msg = SubElement(root, 'message')
192
msg.text = rev.message
349
msg.text = escape_invalid_chars(rev.message)[0]
194
351
if rev.parent_ids:
195
352
pelts = SubElement(root, 'parents')
196
353
pelts.tail = pelts.text = '\n'
197
354
for parent_id in rev.parent_ids:
198
assert isinstance(parent_id, basestring)
355
_mod_revision.check_not_reserved_id(parent_id)
199
356
p = SubElement(pelts, 'revision_ref')
358
if isinstance(parent_id, str):
359
parent_id = decode_utf8(parent_id)
201
360
p.set('revision_id', parent_id)
202
361
if rev.properties:
203
362
self._pack_revision_properties(rev, root)
206
365
def _pack_revision_properties(self, rev, under_element):
207
366
top_elt = SubElement(under_element, 'properties')
208
367
for prop_name, prop_value in sorted(rev.properties.items()):
209
assert isinstance(prop_name, basestring)
210
assert isinstance(prop_value, basestring)
211
368
prop_elt = SubElement(top_elt, 'property')
212
369
prop_elt.set('name', prop_name)
213
370
prop_elt.text = prop_value
214
371
prop_elt.tail = '\n'
215
372
top_elt.tail = '\n'
217
def _unpack_inventory(self, elt):
218
"""Construct from XML Element
220
assert elt.tag == 'inventory'
221
root_id = elt.get('file_id') or ROOT_ID
374
def _unpack_inventory(self, elt, revision_id=None, entry_cache=None):
375
"""Construct from XML Element"""
376
if elt.tag != 'inventory':
377
raise errors.UnexpectedInventoryFormat('Root tag is %r' % elt.tag)
222
378
format = elt.get('format')
223
if format is not None:
225
raise BzrError("invalid format version %r on inventory"
379
if format != self.format_num:
380
raise errors.UnexpectedInventoryFormat('Invalid format version %r'
227
382
revision_id = elt.get('revision_id')
228
383
if revision_id is not None:
229
revision_id = cache_utf8.get_cached_unicode(revision_id)
230
inv = Inventory(root_id, revision_id=revision_id)
384
revision_id = cache_utf8.encode(revision_id)
385
inv = inventory.Inventory(root_id=None, revision_id=revision_id)
232
ie = self._unpack_entry(e)
233
if ie.parent_id == ROOT_ID:
234
ie.parent_id = root_id
387
ie = self._unpack_entry(e, entry_cache=entry_cache)
389
self._check_cache_size(len(inv), entry_cache)
238
def _unpack_entry(self, elt, none_parents=False):
392
def _unpack_entry(self, elt, entry_cache=None):
394
file_id = elt_get('file_id')
395
revision = elt_get('revision')
396
# Check and see if we have already unpacked this exact entry
397
# Some timings for "repo.revision_trees(last_100_revs)"
399
# unmodified 4.1s 40.8s
401
# using fifo 2.83s 29.1s
405
# no_copy 2.00s 20.5s
406
# no_c,dict 1.95s 18.0s
407
# Note that a cache of 10k nodes is more than sufficient to hold all of
408
# the inventory for the last 100 revs for bzr, but not for mysql (20k
409
# is enough for mysql, which saves the same 2s as using a dict)
411
# Breakdown of mysql using time.clock()
412
# 4.1s 2 calls to element.get for file_id, revision_id
413
# 4.5s cache_hit lookup
414
# 7.1s InventoryFile.copy()
415
# 2.4s InventoryDirectory.copy()
416
# 0.4s decoding unique entries
417
# 1.6s decoding entries after FIFO fills up
418
# 0.8s Adding nodes to FIFO (including flushes)
419
# 0.1s cache miss lookups
421
# 4.1s 2 calls to element.get for file_id, revision_id
422
# 9.9s cache_hit lookup
423
# 10.8s InventoryEntry.copy()
424
# 0.3s cache miss lookus
425
# 1.2s decoding entries
426
# 1.0s adding nodes to LRU
427
if entry_cache is not None and revision is not None:
428
key = (file_id, revision)
430
# We copy it, because some operations may mutate it
431
cached_ie = entry_cache[key]
435
# Only copying directory entries drops us 2.85s => 2.35s
436
# if cached_ie.kind == 'directory':
437
# return cached_ie.copy()
439
return cached_ie.copy()
240
442
if not InventoryEntry.versionable_kind(kind):
241
443
raise AssertionError('unsupported entry kind %s' % kind)
243
get_cached = cache_utf8.get_cached_unicode
445
get_cached = _get_utf8_or_ascii
245
parent_id = elt.get('parent_id')
246
if parent_id is None and not none_parents:
248
# TODO: jam 20060817 At present, caching file ids costs us too
249
# much time. It slows down overall read performances from
250
# approx 500ms to 700ms. And doesn't improve future reads.
251
# it might be because revision ids and file ids are mixing.
252
# Consider caching *just* the file ids, for a limited period
254
#parent_id = get_cached(parent_id)
255
#file_id = get_cached(elt.get('file_id'))
256
file_id = elt.get('file_id')
447
file_id = get_cached(file_id)
448
if revision is not None:
449
revision = get_cached(revision)
450
parent_id = elt_get('parent_id')
451
if parent_id is not None:
452
parent_id = get_cached(parent_id)
258
454
if kind == 'directory':
259
455
ie = inventory.InventoryDirectory(file_id,
262
458
elif kind == 'file':
263
459
ie = inventory.InventoryFile(file_id,
266
ie.text_sha1 = elt.get('text_sha1')
267
if elt.get('executable') == 'yes':
462
ie.text_sha1 = elt_get('text_sha1')
463
if elt_get('executable') == 'yes':
268
464
ie.executable = True
269
v = elt.get('text_size')
465
v = elt_get('text_size')
270
466
ie.text_size = v and int(v)
271
467
elif kind == 'symlink':
272
468
ie = inventory.InventoryLink(file_id,
275
ie.symlink_target = elt.get('symlink_target')
471
ie.symlink_target = elt_get('symlink_target')
277
raise BzrError("unknown kind %r" % kind)
278
revision = elt.get('revision')
279
if revision is not None:
280
revision = get_cached(revision)
473
raise errors.UnsupportedInventoryKind(kind)
281
474
ie.revision = revision
475
if revision is not None and entry_cache is not None:
476
# We cache a copy() because callers like to mutate objects, and
477
# that would cause the item in cache to mutate as well.
478
# This has a small effect on many-inventory performance, because
479
# the majority fraction is spent in cache hits, not misses.
480
entry_cache[key] = ie.copy()
285
484
def _unpack_revision(self, elt):
286
485
"""XML Element -> Revision object"""
287
assert elt.tag == 'revision'
288
486
format = elt.get('format')
487
format_num = self.format_num
488
if self.revision_format_num is not None:
489
format_num = self.revision_format_num
289
490
if format is not None:
291
raise BzrError("invalid format version %r on inventory"
491
if format != format_num:
492
raise BzrError("invalid format version %r on revision"
293
get_cached = cache_utf8.get_cached_unicode
494
get_cached = _get_utf8_or_ascii
294
495
rev = Revision(committer = elt.get('committer'),
295
496
timestamp = float(elt.get('timestamp')),
296
497
revision_id = get_cached(elt.get('revision_id')),
added
removed
bzrlib/xml8.py
