1
# Copyright (C) 2005-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22
revision as _mod_revision,
25
from bzrlib.xml_serializer import (
32
serialize_inventory_flat,
33
unpack_inventory_entry,
34
unpack_inventory_flat,
36
from bzrlib.revision import Revision
37
from bzrlib.errors import BzrError
49
def _unescaper(match, _map=_xml_unescape_map):
54
if not code.startswith('#'):
56
return unichr(int(code[1:])).encode('utf8')
59
_unescape_re = lazy_regex.lazy_compile('\&([^;]*);')
61
def _unescape_xml(data):
62
"""Unescape predefined XML entities in a string of data."""
63
return _unescape_re.sub(_unescaper, data)
66
class Serializer_v8(XMLSerializer):
67
"""This serialiser adds rich roots.
69
Its revision format number matches its inventory number.
75
support_altered_by_hack = True
76
# This format supports the altered-by hack that reads file ids directly out
77
# of the versionedfile, without doing XML parsing.
79
supported_kinds = set(['file', 'directory', 'symlink'])
81
revision_format_num = None
83
# The search regex used by xml based repositories to determine what things
84
# where changed in a single commit.
85
_file_ids_altered_regex = lazy_regex.lazy_compile(
86
r'file_id="(?P<file_id>[^"]+)"'
87
r'.* revision="(?P<revision_id>[^"]+)"'
90
def _check_revisions(self, inv):
91
"""Extension point for subclasses to check during serialisation.
93
:param inv: An inventory about to be serialised, to be checked.
94
:raises: AssertionError if an error has occurred.
96
if inv.revision_id is None:
97
raise AssertionError("inv.revision_id is None")
98
if inv.root.revision is None:
99
raise AssertionError("inv.root.revision is None")
101
def _check_cache_size(self, inv_size, entry_cache):
102
"""Check that the entry_cache is large enough.
104
We want the cache to be ~2x the size of an inventory. The reason is
105
because we use a FIFO cache, and how Inventory records are likely to
106
change. In general, you have a small number of records which change
107
often, and a lot of records which do not change at all. So when the
108
cache gets full, you actually flush out a lot of the records you are
109
interested in, which means you need to recreate all of those records.
110
An LRU Cache would be better, but the overhead negates the cache
113
One way to look at it, only the size of the cache > len(inv) is your
114
'working' set. And in general, it shouldn't be a problem to hold 2
115
inventories in memory anyway.
117
:param inv_size: The number of entries in an inventory.
119
if entry_cache is None:
121
# 1.5 times might also be reasonable.
122
recommended_min_cache_size = inv_size * 1.5
123
if entry_cache.cache_size() < recommended_min_cache_size:
124
recommended_cache_size = inv_size * 2
125
trace.mutter('Resizing the inventory entry cache from %d to %d',
126
entry_cache.cache_size(), recommended_cache_size)
127
entry_cache.resize(recommended_cache_size)
129
def write_inventory_to_lines(self, inv):
130
"""Return a list of lines with the encoded inventory."""
131
return self.write_inventory(inv, None)
133
def write_inventory_to_string(self, inv, working=False):
134
"""Just call write_inventory with a StringIO and return the value.
136
:param working: If True skip history data - text_sha1, text_size,
137
reference_revision, symlink_target.
139
sio = cStringIO.StringIO()
140
self.write_inventory(inv, sio, working)
141
return sio.getvalue()
143
def write_inventory(self, inv, f, working=False):
144
"""Write inventory to a file.
146
:param inv: the inventory to write.
147
:param f: the file to write. (May be None if the lines are the desired
149
:param working: If True skip history data - text_sha1, text_size,
150
reference_revision, symlink_target.
151
:return: The inventory as a list of lines.
154
append = output.append
155
self._append_inventory_root(append, inv)
156
serialize_inventory_flat(inv, append,
157
self.root_id, self.supported_kinds, working)
160
# Just to keep the cache from growing without bounds
161
# but we may actually not want to do clear the cache
165
def _append_inventory_root(self, append, inv):
166
"""Append the inventory root to output."""
167
if inv.revision_id is not None:
168
revid1 = ' revision_id="'
169
revid2 = encode_and_escape(inv.revision_id)
173
append('<inventory format="%s"%s%s>\n' % (
174
self.format_num, revid1, revid2))
175
append('<directory file_id="%s name="%s revision="%s />\n' % (
176
encode_and_escape(inv.root.file_id),
177
encode_and_escape(inv.root.name),
178
encode_and_escape(inv.root.revision)))
180
def _pack_revision(self, rev):
181
"""Revision object -> xml tree"""
182
# For the XML format, we need to write them as Unicode rather than as
183
# utf-8 strings. So that cElementTree can handle properly escaping
185
decode_utf8 = cache_utf8.decode
186
revision_id = rev.revision_id
187
if isinstance(revision_id, str):
188
revision_id = decode_utf8(revision_id)
189
format_num = self.format_num
190
if self.revision_format_num is not None:
191
format_num = self.revision_format_num
192
root = Element('revision',
193
committer = rev.committer,
194
timestamp = '%.3f' % rev.timestamp,
195
revision_id = revision_id,
196
inventory_sha1 = rev.inventory_sha1,
199
if rev.timezone is not None:
200
root.set('timezone', str(rev.timezone))
202
msg = SubElement(root, 'message')
203
msg.text = escape_invalid_chars(rev.message)[0]
206
pelts = SubElement(root, 'parents')
207
pelts.tail = pelts.text = '\n'
208
for parent_id in rev.parent_ids:
209
_mod_revision.check_not_reserved_id(parent_id)
210
p = SubElement(pelts, 'revision_ref')
212
if isinstance(parent_id, str):
213
parent_id = decode_utf8(parent_id)
214
p.set('revision_id', parent_id)
216
self._pack_revision_properties(rev, root)
219
def _pack_revision_properties(self, rev, under_element):
220
top_elt = SubElement(under_element, 'properties')
221
for prop_name, prop_value in sorted(rev.properties.items()):
222
prop_elt = SubElement(top_elt, 'property')
223
prop_elt.set('name', prop_name)
224
prop_elt.text = prop_value
228
def _unpack_entry(self, elt, entry_cache=None, return_from_cache=False):
229
# This is here because it's overridden by xml7
230
return unpack_inventory_entry(elt, entry_cache,
233
def _unpack_inventory(self, elt, revision_id=None, entry_cache=None,
234
return_from_cache=False):
235
"""Construct from XML Element"""
236
inv = unpack_inventory_flat(elt, self.format_num, self._unpack_entry,
237
entry_cache, return_from_cache)
238
self._check_cache_size(len(inv), entry_cache)
241
def _unpack_revision(self, elt):
242
"""XML Element -> Revision object"""
243
format = elt.get('format')
244
format_num = self.format_num
245
if self.revision_format_num is not None:
246
format_num = self.revision_format_num
247
if format is not None:
248
if format != format_num:
249
raise BzrError("invalid format version %r on revision"
251
get_cached = get_utf8_or_ascii
252
rev = Revision(committer = elt.get('committer'),
253
timestamp = float(elt.get('timestamp')),
254
revision_id = get_cached(elt.get('revision_id')),
255
inventory_sha1 = elt.get('inventory_sha1')
257
parents = elt.find('parents') or []
259
rev.parent_ids.append(get_cached(p.get('revision_id')))
260
self._unpack_revision_properties(elt, rev)
261
v = elt.get('timezone')
265
rev.timezone = int(v)
266
rev.message = elt.findtext('message') # text of <message>
269
def _unpack_revision_properties(self, elt, rev):
270
"""Unpack properties onto a revision."""
271
props_elt = elt.find('properties')
274
for prop_elt in props_elt:
275
if prop_elt.tag != 'property':
276
raise AssertionError(
277
"bad tag under properties list: %r" % prop_elt.tag)
278
name = prop_elt.get('name')
279
value = prop_elt.text
280
# If a property had an empty value ('') cElementTree reads
281
# that back as None, convert it back to '', so that all
282
# properties have string values
285
if name in rev.properties:
286
raise AssertionError("repeated property %r" % name)
287
rev.properties[name] = value
289
def _find_text_key_references(self, line_iterator):
290
"""Core routine for extracting references to texts from inventories.
292
This performs the translation of xml lines to revision ids.
294
:param line_iterator: An iterator of lines, origin_version_id
295
:return: A dictionary mapping text keys ((fileid, revision_id) tuples)
296
to whether they were referred to by the inventory of the
297
revision_id that they contain. Note that if that revision_id was
298
not part of the line_iterator's output then False will be given -
299
even though it may actually refer to that key.
301
if not self.support_altered_by_hack:
302
raise AssertionError(
303
"_find_text_key_references only "
304
"supported for branches which store inventory as unnested xml"
305
", not on %r" % self)
308
# this code needs to read every new line in every inventory for the
309
# inventories [revision_ids]. Seeing a line twice is ok. Seeing a line
310
# not present in one of those inventories is unnecessary but not
311
# harmful because we are filtering by the revision id marker in the
312
# inventory lines : we only select file ids altered in one of those
313
# revisions. We don't need to see all lines in the inventory because
314
# only those added in an inventory in rev X can contain a revision=X
316
unescape_revid_cache = {}
317
unescape_fileid_cache = {}
319
# jam 20061218 In a big fetch, this handles hundreds of thousands
320
# of lines, so it has had a lot of inlining and optimizing done.
321
# Sorry that it is a little bit messy.
322
# Move several functions to be local variables, since this is a long
324
search = self._file_ids_altered_regex.search
325
unescape = _unescape_xml
326
setdefault = result.setdefault
327
for line, line_key in line_iterator:
331
# One call to match.group() returning multiple items is quite a
332
# bit faster than 2 calls to match.group() each returning 1
333
file_id, revision_id = match.group('file_id', 'revision_id')
335
# Inlining the cache lookups helps a lot when you make 170,000
336
# lines and 350k ids, versus 8.4 unique ids.
337
# Using a cache helps in 2 ways:
338
# 1) Avoids unnecessary decoding calls
339
# 2) Re-uses cached strings, which helps in future set and
341
# (2) is enough that removing encoding entirely along with
342
# the cache (so we are using plain strings) results in no
343
# performance improvement.
345
revision_id = unescape_revid_cache[revision_id]
347
unescaped = unescape(revision_id)
348
unescape_revid_cache[revision_id] = unescaped
349
revision_id = unescaped
351
# Note that unconditionally unescaping means that we deserialise
352
# every fileid, which for general 'pull' is not great, but we don't
353
# really want to have some many fulltexts that this matters anyway.
356
file_id = unescape_fileid_cache[file_id]
358
unescaped = unescape(file_id)
359
unescape_fileid_cache[file_id] = unescaped
362
key = (file_id, revision_id)
363
setdefault(key, False)
364
if revision_id == line_key[-1]:
369
serializer_v8 = Serializer_v8()
added
removed
bzrlib/xml8.py
