13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
"""XML externalization support."""
19
from __future__ import absolute_import
21
19
# "XML is like violence: if it doesn't solve your problem, you aren't
22
20
# using enough of it." -- various
24
22
# importing this module is fairly slow because it has to load several
25
from bzrlib.trace import mutter, warning
30
import xml.etree.cElementTree as elementtree
31
ParseError = getattr(elementtree, "ParseError", SyntaxError)
29
# it's in this package in python2.5
30
from xml.etree.cElementTree import (ElementTree, SubElement, Element,
31
XMLTreeBuilder, fromstring, tostring)
32
import xml.etree as elementtree
34
from cElementTree import (ElementTree, SubElement, Element,
35
XMLTreeBuilder, fromstring, tostring)
37
ParseError = SyntaxError
32
38
except ImportError:
33
# Fall back to pure python implementation if C extension is unavailable
34
import xml.etree.ElementTree as elementtree
36
from xml.etree.ElementTree import ParseError
38
from xml.parsers.expat import ExpatError as ParseError
40
(ElementTree, SubElement, Element, fromstring, tostring) = (
41
elementtree.ElementTree, elementtree.SubElement, elementtree.Element,
42
elementtree.fromstring, elementtree.tostring)
50
from ..sixish import text_type, bytesintern
57
class XMLSerializer(serializer.Serializer):
58
"""Abstract XML object serialize/deserialize"""
60
squashes_xml_invalid_characters = True
62
def read_inventory_from_string(self, xml_string, revision_id=None,
63
entry_cache=None, return_from_cache=False):
64
"""Read xml_string into an inventory object.
66
:param xml_string: The xml to read.
67
:param revision_id: If not-None, the expected revision id of the
68
inventory. Some serialisers use this to set the results' root
69
revision. This should be supplied for deserialising all
70
from-repository inventories so that xml5 inventories that were
71
serialised without a revision identifier can be given the right
72
revision id (but not for working tree inventories where users can
73
edit the data without triggering checksum errors or anything).
74
:param entry_cache: An optional cache of InventoryEntry objects. If
75
supplied we will look up entries via (file_id, revision_id) which
76
should map to a valid InventoryEntry (File/Directory/etc) object.
77
:param return_from_cache: Return entries directly from the cache,
78
rather than copying them first. This is only safe if the caller
79
promises not to mutate the returned inventory entries, but it can
80
make some operations significantly faster.
39
mutter('WARNING: using slower ElementTree; consider installing cElementTree'
40
" and make sure it's on your PYTHONPATH")
41
# this copy is shipped with bzr
42
from util.elementtree.ElementTree import (ElementTree, SubElement,
43
Element, XMLTreeBuilder,
45
import util.elementtree as elementtree
46
from xml.parsers.expat import ExpatError as ParseError
48
from bzrlib import errors
51
class Serializer(object):
52
"""Abstract object serialize/deserialize"""
53
def write_inventory(self, inv, f):
54
"""Write inventory to a file"""
55
elt = self._pack_inventory(inv)
56
self._write_element(elt, f)
58
def write_inventory_to_string(self, inv):
59
return tostring(self._pack_inventory(inv)) + '\n'
61
def read_inventory_from_string(self, xml_string):
83
return self._unpack_inventory(fromstring(xml_string), revision_id,
84
entry_cache=entry_cache,
85
return_from_cache=return_from_cache)
86
except ParseError as e:
63
return self._unpack_inventory(fromstring(xml_string))
87
65
raise errors.UnexpectedInventoryFormat(e)
89
def read_inventory(self, f, revision_id=None):
67
def read_inventory(self, f):
92
return self._unpack_inventory(self._read_element(f),
96
except ParseError as e:
69
return self._unpack_inventory(self._read_element(f))
97
71
raise errors.UnexpectedInventoryFormat(e)
99
73
def write_revision(self, rev, f):
100
74
self._write_element(self._pack_revision(rev), f)
102
76
def write_revision_to_string(self, rev):
103
return tostring(self._pack_revision(rev)) + b'\n'
77
return tostring(self._pack_revision(rev)) + '\n'
105
79
def read_revision(self, f):
106
80
return self._unpack_revision(self._read_element(f))
111
85
def _write_element(self, elt, f):
112
86
ElementTree(elt).write(f, 'utf-8')
115
89
def _read_element(self, f):
116
90
return ElementTree().parse(f)
119
def escape_invalid_chars(message):
120
"""Escape the XML-invalid characters in a commit message.
122
:param message: Commit message to escape
123
:return: tuple with escaped message and number of characters escaped
127
# Python strings can include characters that can't be
128
# represented in well-formed XML; escape characters that
129
# aren't listed in the XML specification
130
# (http://www.w3.org/TR/REC-xml/#NT-Char).
131
return re.subn(u'[^\x09\x0A\x0D\u0020-\uD7FF\uE000-\uFFFD]+',
132
lambda match: match.group(0).encode('unicode_escape'),
136
def get_utf8_or_ascii(a_str, _encode_utf8=cache_utf8.encode):
137
"""Return a cached version of the string.
139
cElementTree will return a plain string if the XML is plain ascii. It only
140
returns Unicode when it needs to. We want to work in utf-8 strings. So if
141
cElementTree returns a plain string, we can just return the cached version.
142
If it is Unicode, then we need to encode it.
144
:param a_str: An 8-bit string or Unicode as returned by
145
cElementTree.Element.get()
146
:return: A utf-8 encoded 8-bit string.
148
# This is fairly optimized because we know what cElementTree does, this is
149
# not meant as a generic function for all cases. Because it is possible for
150
# an 8-bit string to not be ascii or valid utf8.
151
if a_str.__class__ is text_type:
152
return _encode_utf8(a_str)
154
return bytesintern(a_str)
157
_utf8_re = lazy_regex.lazy_compile(b'[&<>\'\"]|[\x80-\xff]+')
158
_unicode_re = lazy_regex.lazy_compile(u'[&<>\'\"\u0080-\uffff]')
163
"'": "'", # FIXME: overkill
170
def _unicode_escape_replace(match, _map=_xml_escape_map):
171
"""Replace a string of non-ascii, non XML safe characters with their escape
173
This will escape both Standard XML escapes, like <>"', etc.
174
As well as escaping non ascii characters, because ElementTree did.
175
This helps us remain compatible to older versions of bzr. We may change
176
our policy in the future, though.
178
# jam 20060816 Benchmarks show that try/KeyError is faster if you
179
# expect the entity to rarely miss. There is about a 10% difference
180
# in overall time. But if you miss frequently, then if None is much
181
# faster. For our use case, we *rarely* have a revision id, file id
182
# or path name that is unicode. So use try/KeyError.
184
return _map[match.group()]
186
return "&#%d;" % ord(match.group())
189
def _utf8_escape_replace(match, _map=_xml_escape_map):
190
"""Escape utf8 characters into XML safe ones.
192
This uses 2 tricks. It is either escaping "standard" characters, like "&<>,
193
or it is handling characters with the high-bit set. For ascii characters,
194
we just lookup the replacement in the dictionary. For everything else, we
195
decode back into Unicode, and then use the XML escape code.
198
return _map[match.group().decode('ascii', 'replace')].encode()
200
return b''.join(b'&#%d;' % ord(uni_chr)
201
for uni_chr in match.group().decode('utf8'))
206
def encode_and_escape(unicode_or_utf8_str, _map=_to_escaped_map):
207
"""Encode the string into utf8, and escape invalid XML characters"""
208
# We frequently get entities we have not seen before, so it is better
209
# to check if None, rather than try/KeyError
210
text = _map.get(unicode_or_utf8_str)
212
if isinstance(unicode_or_utf8_str, text_type):
213
# The alternative policy is to do a regular UTF8 encoding
214
# and then escape only XML meta characters.
215
# Performance is equivalent once you use cache_utf8. *However*
216
# this makes the serialized texts incompatible with old versions
217
# of bzr. So no net gain. (Perhaps the read code would handle utf8
218
# better than entity escapes, but cElementTree seems to do just fine
220
text = _unicode_re.sub(_unicode_escape_replace, unicode_or_utf8_str).encode() + b'"'
222
# Plain strings are considered to already be in utf-8 so we do a
223
# slightly different method for escaping.
224
text = _utf8_re.sub(_utf8_escape_replace,
225
unicode_or_utf8_str) + b'"'
226
_map[unicode_or_utf8_str] = text
231
"""Clean out the unicode => escaped map"""
232
_to_escaped_map.clear()
235
def unpack_inventory_entry(elt, entry_cache=None, return_from_cache=False):
237
file_id = elt_get('file_id')
238
revision = elt_get('revision')
239
# Check and see if we have already unpacked this exact entry
240
# Some timings for "repo.revision_trees(last_100_revs)"
242
# unmodified 4.1s 40.8s
244
# using fifo 2.83s 29.1s
248
# no_copy 2.00s 20.5s
249
# no_c,dict 1.95s 18.0s
250
# Note that a cache of 10k nodes is more than sufficient to hold all of
251
# the inventory for the last 100 revs for bzr, but not for mysql (20k
252
# is enough for mysql, which saves the same 2s as using a dict)
254
# Breakdown of mysql using time.clock()
255
# 4.1s 2 calls to element.get for file_id, revision_id
256
# 4.5s cache_hit lookup
257
# 7.1s InventoryFile.copy()
258
# 2.4s InventoryDirectory.copy()
259
# 0.4s decoding unique entries
260
# 1.6s decoding entries after FIFO fills up
261
# 0.8s Adding nodes to FIFO (including flushes)
262
# 0.1s cache miss lookups
264
# 4.1s 2 calls to element.get for file_id, revision_id
265
# 9.9s cache_hit lookup
266
# 10.8s InventoryEntry.copy()
267
# 0.3s cache miss lookus
268
# 1.2s decoding entries
269
# 1.0s adding nodes to LRU
270
if entry_cache is not None and revision is not None:
271
key = (file_id, revision)
273
# We copy it, because some operations may mutate it
274
cached_ie = entry_cache[key]
278
# Only copying directory entries drops us 2.85s => 2.35s
279
if return_from_cache:
280
if cached_ie.kind == 'directory':
281
return cached_ie.copy()
283
return cached_ie.copy()
286
if not inventory.InventoryEntry.versionable_kind(kind):
287
raise AssertionError('unsupported entry kind %s' % kind)
289
file_id = get_utf8_or_ascii(file_id)
290
if revision is not None:
291
revision = get_utf8_or_ascii(revision)
292
parent_id = elt_get('parent_id')
293
if parent_id is not None:
294
parent_id = get_utf8_or_ascii(parent_id)
296
if kind == 'directory':
297
ie = inventory.InventoryDirectory(file_id,
301
ie = inventory.InventoryFile(file_id,
304
ie.text_sha1 = elt_get('text_sha1')
305
if ie.text_sha1 is not None:
306
ie.text_sha1 = ie.text_sha1.encode('ascii')
307
if elt_get('executable') == 'yes':
309
v = elt_get('text_size')
310
ie.text_size = v and int(v)
311
elif kind == 'symlink':
312
ie = inventory.InventoryLink(file_id,
315
ie.symlink_target = elt_get('symlink_target')
316
elif kind == 'tree-reference':
317
file_id = get_utf8_or_ascii(elt.attrib['file_id'])
318
name = elt.attrib['name']
319
parent_id = get_utf8_or_ascii(elt.attrib['parent_id'])
320
revision = get_utf8_or_ascii(elt.get('revision'))
321
reference_revision = get_utf8_or_ascii(elt.get('reference_revision'))
322
ie = inventory.TreeReference(file_id, name, parent_id, revision,
325
raise errors.UnsupportedInventoryKind(kind)
326
ie.revision = revision
327
if revision is not None and entry_cache is not None:
328
# We cache a copy() because callers like to mutate objects, and
329
# that would cause the item in cache to mutate as well.
330
# This has a small effect on many-inventory performance, because
331
# the majority fraction is spent in cache hits, not misses.
332
entry_cache[key] = ie.copy()
337
def unpack_inventory_flat(elt, format_num, unpack_entry,
338
entry_cache=None, return_from_cache=False):
339
"""Unpack a flat XML inventory.
341
:param elt: XML element for the inventory
342
:param format_num: Expected format number
343
:param unpack_entry: Function for unpacking inventory entries
344
:return: An inventory
345
:raise UnexpectedInventoryFormat: When unexpected elements or data is
348
if elt.tag != 'inventory':
349
raise errors.UnexpectedInventoryFormat('Root tag is %r' % elt.tag)
350
format = elt.get('format')
351
if ((format is None and format_num is not None)
352
or format.encode() != format_num):
353
raise errors.UnexpectedInventoryFormat('Invalid format version %r'
355
revision_id = elt.get('revision_id')
356
if revision_id is not None:
357
revision_id = cache_utf8.encode(revision_id)
358
inv = inventory.Inventory(root_id=None, revision_id=revision_id)
360
ie = unpack_entry(e, entry_cache, return_from_cache)
365
def serialize_inventory_flat(inv, append, root_id, supported_kinds, working):
366
"""Serialize an inventory to a flat XML file.
368
:param inv: Inventory to serialize
369
:param append: Function for writing a line of output
370
:param working: If True skip history data - text_sha1, text_size,
371
reference_revision, symlink_target. self._check_revisions(inv)
373
entries = inv.iter_entries()
375
root_path, root_ie = next(entries)
376
for path, ie in entries:
377
if ie.parent_id != root_id:
378
parent_str = b' parent_id="'
379
parent_id = encode_and_escape(ie.parent_id)
383
if ie.kind == 'file':
385
executable = b' executable="yes"'
389
append(b'<file%s file_id="%s name="%s%s%s revision="%s '
390
b'text_sha1="%s" text_size="%d" />\n' % (
391
executable, encode_and_escape(ie.file_id),
392
encode_and_escape(ie.name), parent_str, parent_id,
393
encode_and_escape(ie.revision), ie.text_sha1,
396
append(b'<file%s file_id="%s name="%s%s%s />\n' % (
397
executable, encode_and_escape(ie.file_id),
398
encode_and_escape(ie.name), parent_str, parent_id))
399
elif ie.kind == 'directory':
401
append(b'<directory file_id="%s name="%s%s%s revision="%s '
403
encode_and_escape(ie.file_id),
404
encode_and_escape(ie.name),
405
parent_str, parent_id,
406
encode_and_escape(ie.revision)))
408
append(b'<directory file_id="%s name="%s%s%s />\n' % (
409
encode_and_escape(ie.file_id),
410
encode_and_escape(ie.name),
411
parent_str, parent_id))
412
elif ie.kind == 'symlink':
414
append(b'<symlink file_id="%s name="%s%s%s revision="%s '
415
b'symlink_target="%s />\n' % (
416
encode_and_escape(ie.file_id),
417
encode_and_escape(ie.name),
418
parent_str, parent_id,
419
encode_and_escape(ie.revision),
420
encode_and_escape(ie.symlink_target)))
422
append(b'<symlink file_id="%s name="%s%s%s />\n' % (
423
encode_and_escape(ie.file_id),
424
encode_and_escape(ie.name),
425
parent_str, parent_id))
426
elif ie.kind == 'tree-reference':
427
if ie.kind not in supported_kinds:
428
raise errors.UnsupportedInventoryKind(ie.kind)
430
append(b'<tree-reference file_id="%s name="%s%s%s '
431
b'revision="%s reference_revision="%s />\n' % (
432
encode_and_escape(ie.file_id),
433
encode_and_escape(ie.name),
434
parent_str, parent_id,
435
encode_and_escape(ie.revision),
436
encode_and_escape(ie.reference_revision)))
438
append(b'<tree-reference file_id="%s name="%s%s%s />\n' % (
439
encode_and_escape(ie.file_id),
440
encode_and_escape(ie.name),
441
parent_str, parent_id))
443
raise errors.UnsupportedInventoryKind(ie.kind)
444
append(b'</inventory>\n')
93
# performance tuning for elementree's serialiser. This should be
94
# sent upstream - RBC 20060523.
95
# the functions here are patched into elementtree at runtime.
97
escape_re = re.compile("[&'\"<>]")
100
"'":"'", # FIXME: overkill
105
def _escape_replace(match, map=escape_map):
106
return map[match.group()]
108
def _escape_attrib(text, encoding=None, replace=None):
109
# escape attribute value
113
text = elementtree.ElementTree._encode(text, encoding)
115
return elementtree.ElementTree._encode_entity(text)
117
return escape_re.sub(_escape_replace, text)
119
text = replace(text, "&", "&")
120
text = replace(text, "'", "'") # FIXME: overkill
121
text = replace(text, "\"", """)
122
text = replace(text, "<", "<")
123
text = replace(text, ">", ">")
125
except (TypeError, AttributeError):
126
elementtree.ElementTree._raise_serialization_error(text)
128
elementtree.ElementTree._escape_attrib = _escape_attrib
130
escape_cdata_re = re.compile("[&<>]")
136
def _escape_cdata_replace(match, map=escape_cdata_map):
137
return map[match.group()]
139
def _escape_cdata(text, encoding=None, replace=None):
140
# escape character data
144
text = elementtree.ElementTree._encode(text, encoding)
146
return elementtree.ElementTree._encode_entity(text)
148
return escape_cdata_re.sub(_escape_cdata_replace, text)
150
text = replace(text, "&", "&")
151
text = replace(text, "<", "<")
152
text = replace(text, ">", ">")
154
except (TypeError, AttributeError):
155
elementtree.ElementTree._raise_serialization_error(text)
157
elementtree.ElementTree._escape_cdata = _escape_cdata
added
removed
bzrlib/xml_serializer.py
