17
17
"""XML externalization support."""
19
from __future__ import absolute_import
19
21
# "XML is like violence: if it doesn't solve your problem, you aren't
20
22
# using enough of it." -- various
22
24
# importing this module is fairly slow because it has to load several
25
from bzrlib.serializer import Serializer
26
from bzrlib.trace import mutter
30
import xml.etree.cElementTree as elementtree
31
ParseError = getattr(elementtree, "ParseError", SyntaxError)
33
# Fall back to pure python implementation if C extension is unavailable
34
import xml.etree.ElementTree as elementtree
30
# it's in this package in python2.5
31
from xml.etree.cElementTree import (ElementTree, SubElement, Element,
32
XMLTreeBuilder, fromstring, tostring)
33
import xml.etree as elementtree
36
from xml.etree.ElementTree import ParseError
34
37
except ImportError:
35
from cElementTree import (ElementTree, SubElement, Element,
36
XMLTreeBuilder, fromstring, tostring)
37
import elementtree.ElementTree
38
ParseError = SyntaxError
40
mutter('WARNING: using slower ElementTree; consider installing cElementTree'
41
" and make sure it's on your PYTHONPATH")
42
# this copy is shipped with bzr
43
from util.elementtree.ElementTree import (ElementTree, SubElement,
44
Element, XMLTreeBuilder,
46
import util.elementtree as elementtree
47
from xml.parsers.expat import ExpatError as ParseError
49
from bzrlib import errors
52
class XMLSerializer(Serializer):
38
from xml.parsers.expat import ExpatError as ParseError
40
(ElementTree, SubElement, Element, fromstring, tostring) = (
41
elementtree.ElementTree, elementtree.SubElement, elementtree.Element,
42
elementtree.fromstring, elementtree.tostring)
50
from ..sixish import text_type, bytesintern
57
class XMLSerializer(serializer.Serializer):
53
58
"""Abstract XML object serialize/deserialize"""
55
60
squashes_xml_invalid_characters = True
103
111
def _write_element(self, elt, f):
104
112
ElementTree(elt).write(f, 'utf-8')
107
115
def _read_element(self, f):
108
116
return ElementTree().parse(f)
111
# performance tuning for elementree's serialiser. This should be
112
# sent upstream - RBC 20060523.
113
# the functions here are patched into elementtree at runtime.
115
escape_re = re.compile("[&'\"<>]")
118
"'":"'", # FIXME: overkill
123
def _escape_replace(match, map=escape_map):
124
return map[match.group()]
126
def _escape_attrib(text, encoding=None, replace=None):
127
# escape attribute value
131
text = elementtree.ElementTree._encode(text, encoding)
133
return elementtree.ElementTree._encode_entity(text)
135
return escape_re.sub(_escape_replace, text)
137
text = replace(text, "&", "&")
138
text = replace(text, "'", "'") # FIXME: overkill
139
text = replace(text, "\"", """)
140
text = replace(text, "<", "<")
141
text = replace(text, ">", ">")
143
except (TypeError, AttributeError):
144
elementtree.ElementTree._raise_serialization_error(text)
146
elementtree.ElementTree._escape_attrib = _escape_attrib
148
escape_cdata_re = re.compile("[&<>]")
154
def _escape_cdata_replace(match, map=escape_cdata_map):
155
return map[match.group()]
157
def _escape_cdata(text, encoding=None, replace=None):
158
# escape character data
162
text = elementtree.ElementTree._encode(text, encoding)
164
return elementtree.ElementTree._encode_entity(text)
166
return escape_cdata_re.sub(_escape_cdata_replace, text)
168
text = replace(text, "&", "&")
169
text = replace(text, "<", "<")
170
text = replace(text, ">", ">")
172
except (TypeError, AttributeError):
173
elementtree.ElementTree._raise_serialization_error(text)
175
elementtree.ElementTree._escape_cdata = _escape_cdata
178
119
def escape_invalid_chars(message):
179
120
"""Escape the XML-invalid characters in a commit message.
190
131
return re.subn(u'[^\x09\x0A\x0D\u0020-\uD7FF\uE000-\uFFFD]+',
191
132
lambda match: match.group(0).encode('unicode_escape'),
136
def get_utf8_or_ascii(a_str, _encode_utf8=cache_utf8.encode):
137
"""Return a cached version of the string.
139
cElementTree will return a plain string if the XML is plain ascii. It only
140
returns Unicode when it needs to. We want to work in utf-8 strings. So if
141
cElementTree returns a plain string, we can just return the cached version.
142
If it is Unicode, then we need to encode it.
144
:param a_str: An 8-bit string or Unicode as returned by
145
cElementTree.Element.get()
146
:return: A utf-8 encoded 8-bit string.
148
# This is fairly optimized because we know what cElementTree does, this is
149
# not meant as a generic function for all cases. Because it is possible for
150
# an 8-bit string to not be ascii or valid utf8.
151
if a_str.__class__ is text_type:
152
return _encode_utf8(a_str)
154
return bytesintern(a_str)
157
_utf8_re = lazy_regex.lazy_compile(b'[&<>\'\"]|[\x80-\xff]+')
158
_unicode_re = lazy_regex.lazy_compile(u'[&<>\'\"\u0080-\uffff]')
163
"'": "'", # FIXME: overkill
170
def _unicode_escape_replace(match, _map=_xml_escape_map):
171
"""Replace a string of non-ascii, non XML safe characters with their escape
173
This will escape both Standard XML escapes, like <>"', etc.
174
As well as escaping non ascii characters, because ElementTree did.
175
This helps us remain compatible to older versions of bzr. We may change
176
our policy in the future, though.
178
# jam 20060816 Benchmarks show that try/KeyError is faster if you
179
# expect the entity to rarely miss. There is about a 10% difference
180
# in overall time. But if you miss frequently, then if None is much
181
# faster. For our use case, we *rarely* have a revision id, file id
182
# or path name that is unicode. So use try/KeyError.
184
return _map[match.group()]
186
return "&#%d;" % ord(match.group())
189
def _utf8_escape_replace(match, _map=_xml_escape_map):
190
"""Escape utf8 characters into XML safe ones.
192
This uses 2 tricks. It is either escaping "standard" characters, like "&<>,
193
or it is handling characters with the high-bit set. For ascii characters,
194
we just lookup the replacement in the dictionary. For everything else, we
195
decode back into Unicode, and then use the XML escape code.
198
return _map[match.group().decode('ascii', 'replace')].encode()
200
return b''.join(b'&#%d;' % ord(uni_chr)
201
for uni_chr in match.group().decode('utf8'))
206
def encode_and_escape(unicode_or_utf8_str, _map=_to_escaped_map):
207
"""Encode the string into utf8, and escape invalid XML characters"""
208
# We frequently get entities we have not seen before, so it is better
209
# to check if None, rather than try/KeyError
210
text = _map.get(unicode_or_utf8_str)
212
if isinstance(unicode_or_utf8_str, text_type):
213
# The alternative policy is to do a regular UTF8 encoding
214
# and then escape only XML meta characters.
215
# Performance is equivalent once you use cache_utf8. *However*
216
# this makes the serialized texts incompatible with old versions
217
# of bzr. So no net gain. (Perhaps the read code would handle utf8
218
# better than entity escapes, but cElementTree seems to do just fine
220
text = _unicode_re.sub(_unicode_escape_replace, unicode_or_utf8_str).encode() + b'"'
222
# Plain strings are considered to already be in utf-8 so we do a
223
# slightly different method for escaping.
224
text = _utf8_re.sub(_utf8_escape_replace,
225
unicode_or_utf8_str) + b'"'
226
_map[unicode_or_utf8_str] = text
231
"""Clean out the unicode => escaped map"""
232
_to_escaped_map.clear()
235
def unpack_inventory_entry(elt, entry_cache=None, return_from_cache=False):
237
file_id = elt_get('file_id')
238
revision = elt_get('revision')
239
# Check and see if we have already unpacked this exact entry
240
# Some timings for "repo.revision_trees(last_100_revs)"
242
# unmodified 4.1s 40.8s
244
# using fifo 2.83s 29.1s
248
# no_copy 2.00s 20.5s
249
# no_c,dict 1.95s 18.0s
250
# Note that a cache of 10k nodes is more than sufficient to hold all of
251
# the inventory for the last 100 revs for bzr, but not for mysql (20k
252
# is enough for mysql, which saves the same 2s as using a dict)
254
# Breakdown of mysql using time.clock()
255
# 4.1s 2 calls to element.get for file_id, revision_id
256
# 4.5s cache_hit lookup
257
# 7.1s InventoryFile.copy()
258
# 2.4s InventoryDirectory.copy()
259
# 0.4s decoding unique entries
260
# 1.6s decoding entries after FIFO fills up
261
# 0.8s Adding nodes to FIFO (including flushes)
262
# 0.1s cache miss lookups
264
# 4.1s 2 calls to element.get for file_id, revision_id
265
# 9.9s cache_hit lookup
266
# 10.8s InventoryEntry.copy()
267
# 0.3s cache miss lookus
268
# 1.2s decoding entries
269
# 1.0s adding nodes to LRU
270
if entry_cache is not None and revision is not None:
271
key = (file_id, revision)
273
# We copy it, because some operations may mutate it
274
cached_ie = entry_cache[key]
278
# Only copying directory entries drops us 2.85s => 2.35s
279
if return_from_cache:
280
if cached_ie.kind == 'directory':
281
return cached_ie.copy()
283
return cached_ie.copy()
286
if not inventory.InventoryEntry.versionable_kind(kind):
287
raise AssertionError('unsupported entry kind %s' % kind)
289
file_id = get_utf8_or_ascii(file_id)
290
if revision is not None:
291
revision = get_utf8_or_ascii(revision)
292
parent_id = elt_get('parent_id')
293
if parent_id is not None:
294
parent_id = get_utf8_or_ascii(parent_id)
296
if kind == 'directory':
297
ie = inventory.InventoryDirectory(file_id,
301
ie = inventory.InventoryFile(file_id,
304
ie.text_sha1 = elt_get('text_sha1')
305
if ie.text_sha1 is not None:
306
ie.text_sha1 = ie.text_sha1.encode('ascii')
307
if elt_get('executable') == 'yes':
309
v = elt_get('text_size')
310
ie.text_size = v and int(v)
311
elif kind == 'symlink':
312
ie = inventory.InventoryLink(file_id,
315
ie.symlink_target = elt_get('symlink_target')
316
elif kind == 'tree-reference':
317
file_id = get_utf8_or_ascii(elt.attrib['file_id'])
318
name = elt.attrib['name']
319
parent_id = get_utf8_or_ascii(elt.attrib['parent_id'])
320
revision = get_utf8_or_ascii(elt.get('revision'))
321
reference_revision = get_utf8_or_ascii(elt.get('reference_revision'))
322
ie = inventory.TreeReference(file_id, name, parent_id, revision,
325
raise errors.UnsupportedInventoryKind(kind)
326
ie.revision = revision
327
if revision is not None and entry_cache is not None:
328
# We cache a copy() because callers like to mutate objects, and
329
# that would cause the item in cache to mutate as well.
330
# This has a small effect on many-inventory performance, because
331
# the majority fraction is spent in cache hits, not misses.
332
entry_cache[key] = ie.copy()
337
def unpack_inventory_flat(elt, format_num, unpack_entry,
338
entry_cache=None, return_from_cache=False):
339
"""Unpack a flat XML inventory.
341
:param elt: XML element for the inventory
342
:param format_num: Expected format number
343
:param unpack_entry: Function for unpacking inventory entries
344
:return: An inventory
345
:raise UnexpectedInventoryFormat: When unexpected elements or data is
348
if elt.tag != 'inventory':
349
raise errors.UnexpectedInventoryFormat('Root tag is %r' % elt.tag)
350
format = elt.get('format')
351
if ((format is None and format_num is not None)
352
or format.encode() != format_num):
353
raise errors.UnexpectedInventoryFormat('Invalid format version %r'
355
revision_id = elt.get('revision_id')
356
if revision_id is not None:
357
revision_id = cache_utf8.encode(revision_id)
358
inv = inventory.Inventory(root_id=None, revision_id=revision_id)
360
ie = unpack_entry(e, entry_cache, return_from_cache)
365
def serialize_inventory_flat(inv, append, root_id, supported_kinds, working):
366
"""Serialize an inventory to a flat XML file.
368
:param inv: Inventory to serialize
369
:param append: Function for writing a line of output
370
:param working: If True skip history data - text_sha1, text_size,
371
reference_revision, symlink_target. self._check_revisions(inv)
373
entries = inv.iter_entries()
375
root_path, root_ie = next(entries)
376
for path, ie in entries:
377
if ie.parent_id != root_id:
378
parent_str = b' parent_id="'
379
parent_id = encode_and_escape(ie.parent_id)
383
if ie.kind == 'file':
385
executable = b' executable="yes"'
389
append(b'<file%s file_id="%s name="%s%s%s revision="%s '
390
b'text_sha1="%s" text_size="%d" />\n' % (
391
executable, encode_and_escape(ie.file_id),
392
encode_and_escape(ie.name), parent_str, parent_id,
393
encode_and_escape(ie.revision), ie.text_sha1,
396
append(b'<file%s file_id="%s name="%s%s%s />\n' % (
397
executable, encode_and_escape(ie.file_id),
398
encode_and_escape(ie.name), parent_str, parent_id))
399
elif ie.kind == 'directory':
401
append(b'<directory file_id="%s name="%s%s%s revision="%s '
403
encode_and_escape(ie.file_id),
404
encode_and_escape(ie.name),
405
parent_str, parent_id,
406
encode_and_escape(ie.revision)))
408
append(b'<directory file_id="%s name="%s%s%s />\n' % (
409
encode_and_escape(ie.file_id),
410
encode_and_escape(ie.name),
411
parent_str, parent_id))
412
elif ie.kind == 'symlink':
414
append(b'<symlink file_id="%s name="%s%s%s revision="%s '
415
b'symlink_target="%s />\n' % (
416
encode_and_escape(ie.file_id),
417
encode_and_escape(ie.name),
418
parent_str, parent_id,
419
encode_and_escape(ie.revision),
420
encode_and_escape(ie.symlink_target)))
422
append(b'<symlink file_id="%s name="%s%s%s />\n' % (
423
encode_and_escape(ie.file_id),
424
encode_and_escape(ie.name),
425
parent_str, parent_id))
426
elif ie.kind == 'tree-reference':
427
if ie.kind not in supported_kinds:
428
raise errors.UnsupportedInventoryKind(ie.kind)
430
append(b'<tree-reference file_id="%s name="%s%s%s '
431
b'revision="%s reference_revision="%s />\n' % (
432
encode_and_escape(ie.file_id),
433
encode_and_escape(ie.name),
434
parent_str, parent_id,
435
encode_and_escape(ie.revision),
436
encode_and_escape(ie.reference_revision)))
438
append(b'<tree-reference file_id="%s name="%s%s%s />\n' % (
439
encode_and_escape(ie.file_id),
440
encode_and_escape(ie.name),
441
parent_str, parent_id))
443
raise errors.UnsupportedInventoryKind(ie.kind)
444
append(b'</inventory>\n')