1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25
revision as _mod_revision,
28
from bzrlib.xml_serializer import SubElement, Element, Serializer
29
from bzrlib.inventory import ROOT_ID, Inventory, InventoryEntry
30
from bzrlib.revision import Revision
31
from bzrlib.errors import BzrError
38
"'":"'", # FIXME: overkill
43
# A cache of InventoryEntry objects
44
_entry_cache = fifo_cache.FIFOCache(10*1024)
47
def _ensure_utf8_re():
48
"""Make sure the _utf8_re and _unicode_re regexes have been compiled."""
49
global _utf8_re, _unicode_re
51
_utf8_re = re.compile('[&<>\'\"]|[\x80-\xff]+')
52
if _unicode_re is None:
53
_unicode_re = re.compile(u'[&<>\'\"\u0080-\uffff]')
56
def _unicode_escape_replace(match, _map=_xml_escape_map):
57
"""Replace a string of non-ascii, non XML safe characters with their escape
59
This will escape both Standard XML escapes, like <>"', etc.
60
As well as escaping non ascii characters, because ElementTree did.
61
This helps us remain compatible to older versions of bzr. We may change
62
our policy in the future, though.
64
# jam 20060816 Benchmarks show that try/KeyError is faster if you
65
# expect the entity to rarely miss. There is about a 10% difference
66
# in overall time. But if you miss frequently, then if None is much
67
# faster. For our use case, we *rarely* have a revision id, file id
68
# or path name that is unicode. So use try/KeyError.
70
return _map[match.group()]
72
return "&#%d;" % ord(match.group())
75
def _utf8_escape_replace(match, _map=_xml_escape_map):
76
"""Escape utf8 characters into XML safe ones.
78
This uses 2 tricks. It is either escaping "standard" characters, like "&<>,
79
or it is handling characters with the high-bit set. For ascii characters,
80
we just lookup the replacement in the dictionary. For everything else, we
81
decode back into Unicode, and then use the XML escape code.
84
return _map[match.group()]
86
return ''.join('&#%d;' % ord(uni_chr)
87
for uni_chr in match.group().decode('utf8'))
92
def _encode_and_escape(unicode_or_utf8_str, _map=_to_escaped_map):
93
"""Encode the string into utf8, and escape invalid XML characters"""
94
# We frequently get entities we have not seen before, so it is better
95
# to check if None, rather than try/KeyError
96
text = _map.get(unicode_or_utf8_str)
98
if unicode_or_utf8_str.__class__ == unicode:
99
# The alternative policy is to do a regular UTF8 encoding
100
# and then escape only XML meta characters.
101
# Performance is equivalent once you use cache_utf8. *However*
102
# this makes the serialized texts incompatible with old versions
103
# of bzr. So no net gain. (Perhaps the read code would handle utf8
104
# better than entity escapes, but cElementTree seems to do just fine
106
text = str(_unicode_re.sub(_unicode_escape_replace,
107
unicode_or_utf8_str)) + '"'
109
# Plain strings are considered to already be in utf-8 so we do a
110
# slightly different method for escaping.
111
text = _utf8_re.sub(_utf8_escape_replace,
112
unicode_or_utf8_str) + '"'
113
_map[unicode_or_utf8_str] = text
117
def _get_utf8_or_ascii(a_str,
118
_encode_utf8=cache_utf8.encode,
119
_get_cached_ascii=cache_utf8.get_cached_ascii):
120
"""Return a cached version of the string.
122
cElementTree will return a plain string if the XML is plain ascii. It only
123
returns Unicode when it needs to. We want to work in utf-8 strings. So if
124
cElementTree returns a plain string, we can just return the cached version.
125
If it is Unicode, then we need to encode it.
127
:param a_str: An 8-bit string or Unicode as returned by
128
cElementTree.Element.get()
129
:return: A utf-8 encoded 8-bit string.
131
# This is fairly optimized because we know what cElementTree does, this is
132
# not meant as a generic function for all cases. Because it is possible for
133
# an 8-bit string to not be ascii or valid utf8.
134
if a_str.__class__ == unicode:
135
return _encode_utf8(a_str)
137
return _get_cached_ascii(a_str)
141
"""Clean out the unicode => escaped map"""
142
_to_escaped_map.clear()
145
class Serializer_v8(Serializer):
146
"""This serialiser adds rich roots.
148
Its revision format number matches its inventory number.
154
support_altered_by_hack = True
155
# This format supports the altered-by hack that reads file ids directly out
156
# of the versionedfile, without doing XML parsing.
158
supported_kinds = set(['file', 'directory', 'symlink'])
160
revision_format_num = None
162
def _check_revisions(self, inv):
163
"""Extension point for subclasses to check during serialisation.
165
:param inv: An inventory about to be serialised, to be checked.
166
:raises: AssertionError if an error has occured.
168
if inv.revision_id is None:
169
raise AssertionError()
170
if inv.root.revision is None:
171
raise AssertionError()
173
def write_inventory_to_lines(self, inv):
174
"""Return a list of lines with the encoded inventory."""
175
return self.write_inventory(inv, None)
177
def write_inventory_to_string(self, inv, working=False):
178
"""Just call write_inventory with a StringIO and return the value.
180
:param working: If True skip history data - text_sha1, text_size,
181
reference_revision, symlink_target.
183
sio = cStringIO.StringIO()
184
self.write_inventory(inv, sio, working)
185
return sio.getvalue()
187
def write_inventory(self, inv, f, working=False):
188
"""Write inventory to a file.
190
:param inv: the inventory to write.
191
:param f: the file to write. (May be None if the lines are the desired
193
:param working: If True skip history data - text_sha1, text_size,
194
reference_revision, symlink_target.
195
:return: The inventory as a list of lines.
198
self._check_revisions(inv)
200
append = output.append
201
self._append_inventory_root(append, inv)
202
entries = inv.iter_entries()
204
root_path, root_ie = entries.next()
205
for path, ie in entries:
206
if ie.parent_id != self.root_id:
207
parent_str = ' parent_id="'
208
parent_id = _encode_and_escape(ie.parent_id)
212
if ie.kind == 'file':
214
executable = ' executable="yes"'
218
append('<file%s file_id="%s name="%s%s%s revision="%s '
219
'text_sha1="%s" text_size="%d" />\n' % (
220
executable, _encode_and_escape(ie.file_id),
221
_encode_and_escape(ie.name), parent_str, parent_id,
222
_encode_and_escape(ie.revision), ie.text_sha1,
225
append('<file%s file_id="%s name="%s%s%s />\n' % (
226
executable, _encode_and_escape(ie.file_id),
227
_encode_and_escape(ie.name), parent_str, parent_id))
228
elif ie.kind == 'directory':
230
append('<directory file_id="%s name="%s%s%s revision="%s '
232
_encode_and_escape(ie.file_id),
233
_encode_and_escape(ie.name),
234
parent_str, parent_id,
235
_encode_and_escape(ie.revision)))
237
append('<directory file_id="%s name="%s%s%s />\n' % (
238
_encode_and_escape(ie.file_id),
239
_encode_and_escape(ie.name),
240
parent_str, parent_id))
241
elif ie.kind == 'symlink':
243
append('<symlink file_id="%s name="%s%s%s revision="%s '
244
'symlink_target="%s />\n' % (
245
_encode_and_escape(ie.file_id),
246
_encode_and_escape(ie.name),
247
parent_str, parent_id,
248
_encode_and_escape(ie.revision),
249
_encode_and_escape(ie.symlink_target)))
251
append('<symlink file_id="%s name="%s%s%s />\n' % (
252
_encode_and_escape(ie.file_id),
253
_encode_and_escape(ie.name),
254
parent_str, parent_id))
255
elif ie.kind == 'tree-reference':
256
if ie.kind not in self.supported_kinds:
257
raise errors.UnsupportedInventoryKind(ie.kind)
259
append('<tree-reference file_id="%s name="%s%s%s '
260
'revision="%s reference_revision="%s />\n' % (
261
_encode_and_escape(ie.file_id),
262
_encode_and_escape(ie.name),
263
parent_str, parent_id,
264
_encode_and_escape(ie.revision),
265
_encode_and_escape(ie.reference_revision)))
267
append('<tree-reference file_id="%s name="%s%s%s />\n' % (
268
_encode_and_escape(ie.file_id),
269
_encode_and_escape(ie.name),
270
parent_str, parent_id))
272
raise errors.UnsupportedInventoryKind(ie.kind)
273
append('</inventory>\n')
276
# Just to keep the cache from growing without bounds
277
# but we may actually not want to do clear the cache
281
def _append_inventory_root(self, append, inv):
282
"""Append the inventory root to output."""
283
if inv.revision_id is not None:
284
revid1 = ' revision_id="'
285
revid2 = _encode_and_escape(inv.revision_id)
289
append('<inventory format="%s"%s%s>\n' % (
290
self.format_num, revid1, revid2))
291
append('<directory file_id="%s name="%s revision="%s />\n' % (
292
_encode_and_escape(inv.root.file_id),
293
_encode_and_escape(inv.root.name),
294
_encode_and_escape(inv.root.revision)))
296
def _pack_revision(self, rev):
297
"""Revision object -> xml tree"""
298
# For the XML format, we need to write them as Unicode rather than as
299
# utf-8 strings. So that cElementTree can handle properly escaping
301
decode_utf8 = cache_utf8.decode
302
revision_id = rev.revision_id
303
if isinstance(revision_id, str):
304
revision_id = decode_utf8(revision_id)
305
format_num = self.format_num
306
if self.revision_format_num is not None:
307
format_num = self.revision_format_num
308
root = Element('revision',
309
committer = rev.committer,
310
timestamp = '%.3f' % rev.timestamp,
311
revision_id = revision_id,
312
inventory_sha1 = rev.inventory_sha1,
315
if rev.timezone is not None:
316
root.set('timezone', str(rev.timezone))
318
msg = SubElement(root, 'message')
319
msg.text = rev.message
322
pelts = SubElement(root, 'parents')
323
pelts.tail = pelts.text = '\n'
324
for parent_id in rev.parent_ids:
325
_mod_revision.check_not_reserved_id(parent_id)
326
p = SubElement(pelts, 'revision_ref')
328
if isinstance(parent_id, str):
329
parent_id = decode_utf8(parent_id)
330
p.set('revision_id', parent_id)
332
self._pack_revision_properties(rev, root)
335
def _pack_revision_properties(self, rev, under_element):
336
top_elt = SubElement(under_element, 'properties')
337
for prop_name, prop_value in sorted(rev.properties.items()):
338
prop_elt = SubElement(top_elt, 'property')
339
prop_elt.set('name', prop_name)
340
prop_elt.text = prop_value
344
def _unpack_inventory(self, elt, revision_id=None):
345
"""Construct from XML Element"""
346
if elt.tag != 'inventory':
347
raise errors.UnexpectedInventoryFormat('Root tag is %r' % elt.tag)
348
format = elt.get('format')
349
if format != self.format_num:
350
raise errors.UnexpectedInventoryFormat('Invalid format version %r'
352
revision_id = elt.get('revision_id')
353
if revision_id is not None:
354
revision_id = cache_utf8.encode(revision_id)
355
inv = inventory.Inventory(root_id=None, revision_id=revision_id)
357
ie = self._unpack_entry(e)
361
def _unpack_entry(self, elt):
362
get_cached = _get_utf8_or_ascii
365
file_id = elt_get('file_id')
366
revision = elt_get('revision')
367
# Check and see if we have already unpacked this exact entry
368
# Some timings for "repo.revision_trees(last_100_revs)"
370
# unmodified 4.1s 40.8s
372
# using fifo 2.83s 29.1s
376
# no_copy 2.00s 20.5s
377
# no_c,dict 1.95s 18.0s
378
# Note that a cache of 10k nodes is more than sufficient to hold all of
379
# the inventory for the last 100 revs for bzr, but not for mysql (20k
380
# is enough for mysql, which saves the same 2s as using a dict)
382
# Breakdown of mysql using time.clock()
383
# 4.1s 2 calls to element.get for file_id, revision_id
384
# 4.5s cache_hit lookup
385
# 7.1s InventoryFile.copy()
386
# 2.4s InventoryDirectory.copy()
387
# 0.4s decoding unique entries
388
# 1.6s decoding entries
389
# 0.8s Adding nodes to FIFO (including flushes)
390
# 0.1s cache miss lookups
392
# 4.1s 2 calls to element.get for file_id, revision_id
393
# 9.9s cache_hit lookup
394
# 10.8s InventoryEntry.copy()
395
# 0.3s cache miss lookus
396
# 1.2s decoding entries
397
# 1.0s adding nodes to LRU
398
key = (file_id, revision)
400
# We copy it, because some operatations may mutate it
401
cached_ie = _entry_cache[key]
405
# Only copying directory entries drops us 2.85s => 2.35s
406
# if cached_ie.kind == 'directory':
407
# return cached_ie.copy()
409
return cached_ie.copy()
412
if not InventoryEntry.versionable_kind(kind):
413
raise AssertionError('unsupported entry kind %s' % kind)
415
file_id = get_cached(file_id)
416
if revision is not None:
417
revision = get_cached(revision)
418
parent_id = elt_get('parent_id')
419
if parent_id is not None:
420
parent_id = get_cached(parent_id)
422
if kind == 'directory':
423
ie = inventory.InventoryDirectory(file_id,
427
ie = inventory.InventoryFile(file_id,
430
ie.text_sha1 = elt_get('text_sha1')
431
if elt_get('executable') == 'yes':
433
v = elt_get('text_size')
434
ie.text_size = v and int(v)
435
elif kind == 'symlink':
436
ie = inventory.InventoryLink(file_id,
439
ie.symlink_target = elt_get('symlink_target')
441
raise errors.UnsupportedInventoryKind(kind)
442
ie.revision = revision
443
if revision is not None:
444
_entry_cache[key] = ie
448
def _unpack_revision(self, elt):
449
"""XML Element -> Revision object"""
450
format = elt.get('format')
451
format_num = self.format_num
452
if self.revision_format_num is not None:
453
format_num = self.revision_format_num
454
if format is not None:
455
if format != format_num:
456
raise BzrError("invalid format version %r on revision"
458
get_cached = _get_utf8_or_ascii
459
rev = Revision(committer = elt.get('committer'),
460
timestamp = float(elt.get('timestamp')),
461
revision_id = get_cached(elt.get('revision_id')),
462
inventory_sha1 = elt.get('inventory_sha1')
464
parents = elt.find('parents') or []
466
rev.parent_ids.append(get_cached(p.get('revision_id')))
467
self._unpack_revision_properties(elt, rev)
468
v = elt.get('timezone')
472
rev.timezone = int(v)
473
rev.message = elt.findtext('message') # text of <message>
476
def _unpack_revision_properties(self, elt, rev):
477
"""Unpack properties onto a revision."""
478
props_elt = elt.find('properties')
481
for prop_elt in props_elt:
482
if prop_elt.tag != 'property':
483
raise AssertionError(
484
"bad tag under properties list: %r" % prop_elt.tag)
485
name = prop_elt.get('name')
486
value = prop_elt.text
487
# If a property had an empty value ('') cElementTree reads
488
# that back as None, convert it back to '', so that all
489
# properties have string values
492
if name in rev.properties:
493
raise AssertionError("repeated property %r" % name)
494
rev.properties[name] = value
497
serializer_v8 = Serializer_v8()
added
removed
bzrlib/xml8.py
