1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
# TODO: Some kind of command-line display of revision properties:
18
# perhaps show them in log -v and allow them as options to the commit command.
20
"""Some functions to enable caching the conversion between unicode to utf8"""
22
from __future__ import absolute_import
26
_utf8_encode = codecs.utf_8_encode
27
_utf8_decode = codecs.utf_8_decode
30
def _utf8_decode_with_None(bytestring, _utf8_decode=_utf8_decode):
31
"""wrap _utf8_decode to support None->None for optional strings.
33
Also, only return the Unicode portion, since we don't care about the second
36
if bytestring is None:
39
return _utf8_decode(bytestring)[0]
42
# Map revisions from and to utf8 encoding
43
# Whenever we do an encode/decode operation, we save the result, so that
44
# we don't have to do it again.
45
_unicode_to_utf8_map = {}
46
_utf8_to_unicode_map = {}
49
def encode(unicode_str,
50
_uni_to_utf8=_unicode_to_utf8_map,
51
_utf8_to_uni=_utf8_to_unicode_map,
52
_utf8_encode=_utf8_encode):
53
"""Take this unicode revision id, and get a unicode version"""
54
# If the key is in the cache try/KeyError is 50% faster than
55
# val = dict.get(key), if val is None:
56
# On jam's machine the difference is
59
# Since these are primarily used when iterating over a knit entry
60
# *most* of the time the key will already be in the cache, so use the
63
return _uni_to_utf8[unicode_str]
65
_uni_to_utf8[unicode_str] = utf8_str = _utf8_encode(unicode_str)[0]
66
_utf8_to_uni[utf8_str] = unicode_str
71
_uni_to_utf8=_unicode_to_utf8_map,
72
_utf8_to_uni=_utf8_to_unicode_map,
73
_utf8_decode=_utf8_decode):
74
"""Take a utf8 revision id, and decode it, but cache the result"""
76
return _utf8_to_uni[utf8_str]
78
unicode_str = _utf8_decode(utf8_str)[0]
79
_utf8_to_uni[utf8_str] = unicode_str
80
_uni_to_utf8[unicode_str] = utf8_str
84
def get_cached_unicode(unicode_str):
85
"""Return a cached version of the unicode string.
87
This has a similar idea to that of intern() in that it tries
88
to return a singleton string. Only it works for unicode strings.
90
# This might return the same object, or it might return the cached one
91
# the decode() should just be a hash lookup, because the encode() side
92
# should add the entry to the maps
93
return decode(encode(unicode_str))
96
def get_cached_utf8(utf8_str):
97
"""Return a cached version of the utf-8 string.
99
Get a cached version of this string (similar to intern()).
100
At present, this will be decoded to ensure it is a utf-8 string. In the
101
future this might change to simply caching the string.
103
return encode(decode(utf8_str))
106
def get_cached_ascii(ascii_str,
107
_uni_to_utf8=_unicode_to_utf8_map,
108
_utf8_to_uni=_utf8_to_unicode_map):
109
"""This is a string which is identical in utf-8 and unicode."""
110
# We don't need to do any encoding, but we want _utf8_to_uni to return a
111
# real Unicode string. Unicode and plain strings of this type will have the
112
# same hash, so we can just use it as the key in _uni_to_utf8, but we need
113
# the return value to be different in _utf8_to_uni
114
uni_str = ascii_str.decode('ascii')
115
ascii_str = _uni_to_utf8.setdefault(uni_str, ascii_str)
116
_utf8_to_uni.setdefault(ascii_str, uni_str)
120
def clear_encoding_cache():
121
"""Clear the encoding and decoding caches"""
122
_unicode_to_utf8_map.clear()
123
_utf8_to_unicode_map.clear()
added
removed
breezy/cache_utf8.py
