/brz/remove-bazaar

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/brz/remove-bazaar
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
1
# Copyright (C) 2011 Canonical Ltd
2
#
5820.1.18 by INADA Naoki
Add copyright for some function.
3
# UTextWrapper._handle_long_word, UTextWrapper._wrap_chunks,
5820.1.27 by INADA Naoki
Fix error when fix_sentence_endings=True.
4
# UTextWrapper._fix_sentence_endings, wrap and fill is copied from Python's
5
# textwrap module (under PSF license) and modified for support CJK.
5820.1.18 by INADA Naoki
Add copyright for some function.
6
# Original Copyright for these functions:
7
#
8
# Copyright (C) 1999-2001 Gregory P. Ward.
9
# Copyright (C) 2002, 2003 Python Software Foundation.
10
#
11
# Written by Greg Ward <gward@python.net>
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
12
# This program is free software; you can redistribute it and/or modify
13
# it under the terms of the GNU General Public License as published by
14
# the Free Software Foundation; either version 2 of the License, or
15
# (at your option) any later version.
16
#
17
# This program is distributed in the hope that it will be useful,
18
# but WITHOUT ANY WARRANTY; without even the implied warranty of
19
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20
# GNU General Public License for more details.
21
#
22
# You should have received a copy of the GNU General Public License
23
# along with this program; if not, write to the Free Software
24
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25
6379.6.3 by Jelmer Vernooij
Use absolute_import.
26
from __future__ import absolute_import
27
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
28
import textwrap
29
from unicodedata import east_asian_width as _eawidth
30
6624 by Jelmer Vernooij
Merge Python3 porting work ('py3 pokes')
31
from . import osutils
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
32
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
33
__all__ = ["UTextWrapper", "fill", "wrap"]
34
7143.15.2 by Jelmer Vernooij
Run autopep8.
35
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
36
class UTextWrapper(textwrap.TextWrapper):
37
    """
38
    Extend TextWrapper for Unicode.
39
40
    This textwrapper handles east asian double width and split word
41
    even if !break_long_words when word contains double width
42
    characters.
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
43
44
    :param ambiguous_width: (keyword argument) width for character when
45
                            unicodedata.east_asian_width(c) == 'A'
5820.1.21 by INADA Naoki
Change default value of ambiguous_width from 2 to 1.
46
                            (default: 1)
5820.1.22 by INADA Naoki
Add document of some limitations in docstring.
47
48
    Limitations:
49
    * expand_tabs doesn't fixed. It uses len() for calculating width
50
      of string on left of TAB.
51
    * Handles one codeunit as a single character having 1 or 2 width.
52
      This is not correct when there are surrogate pairs, combined
53
      characters or zero-width characters.
54
    * Treats all asian character are line breakable. But it is not
55
      true because line breaking is prohibited around some characters.
56
      (For example, breaking before punctation mark is prohibited.)
57
      See UAX # 14 "UNICODE LINE BREAKING ALGORITHM"
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
58
    """
5820.1.15 by Martin
Cope with lack of TextWrapper.drop_whitespace before Python 2.6
59
5820.1.9 by INADA Naoki
Default width of UTextWrapper is also osutils.terminal_widtth() and
60
    def __init__(self, width=None, **kwargs):
61
        if width is None:
62
            width = (osutils.terminal_width() or
7143.15.2 by Jelmer Vernooij
Run autopep8.
63
                     osutils.default_terminal_width) - 1
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
64
5820.1.21 by INADA Naoki
Change default value of ambiguous_width from 2 to 1.
65
        ambi_width = kwargs.pop('ambiguous_width', 1)
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
66
        if ambi_width == 1:
67
            self._east_asian_doublewidth = 'FW'
68
        elif ambi_width == 2:
69
            self._east_asian_doublewidth = 'FWA'
70
        else:
71
            raise ValueError("ambiguous_width should be 1 or 2")
72
7112.1.2 by Jelmer Vernooij
Fix test on python2.
73
        self.max_lines = kwargs.get('max_lines', None)
5820.1.9 by INADA Naoki
Default width of UTextWrapper is also osutils.terminal_widtth() and
74
        textwrap.TextWrapper.__init__(self, width, **kwargs)
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
75
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
76
    def _unicode_char_width(self, uc):
77
        """Return width of character `uc`.
78
79
        :param:     uc      Single unicode character.
80
        """
81
        # 'A' means width of the character is not be able to determine.
82
        # We assume that it's width is 2 because longer wrap may over
83
        # terminal width but shorter wrap may be acceptable.
84
        return (_eawidth(uc) in self._east_asian_doublewidth and 2) or 1
85
86
    def _width(self, s):
87
        """Returns width for s.
5820.1.26 by INADA Naoki
Cleanup. Remove spaces in empty line and shorten line having 80 characters.
88
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
89
        When s is unicode, take care of east asian width.
90
        When s is bytes, treat all byte is single width character.
91
        """
92
        charwidth = self._unicode_char_width
93
        return sum(charwidth(c) for c in s)
94
95
    def _cut(self, s, width):
96
        """Returns head and rest of s. (head+rest == s)
97
98
        Head is large as long as _width(head) <= width.
99
        """
100
        w = 0
101
        charwidth = self._unicode_char_width
102
        for pos, c in enumerate(s):
103
            w += charwidth(c)
104
            if w > width:
105
                return s[:pos], s[pos:]
106
        return s, u''
107
5820.1.27 by INADA Naoki
Fix error when fix_sentence_endings=True.
108
    def _fix_sentence_endings(self, chunks):
109
        """_fix_sentence_endings(chunks : [string])
110
111
        Correct for sentence endings buried in 'chunks'.  Eg. when the
112
        original text contains "... foo.\nBar ...", munge_whitespace()
113
        and split() will convert that to [..., "foo.", " ", "Bar", ...]
114
        which has one too few spaces; this method simply changes the one
115
        space to two.
116
117
        Note: This function is copied from textwrap.TextWrap and modified
118
        to use unicode always.
119
        """
120
        i = 0
7143.15.2 by Jelmer Vernooij
Run autopep8.
121
        L = len(chunks) - 1
5820.1.27 by INADA Naoki
Fix error when fix_sentence_endings=True.
122
        patsearch = self.sentence_end_re.search
123
        while i < L:
7143.15.2 by Jelmer Vernooij
Run autopep8.
124
            if chunks[i + 1] == u" " and patsearch(chunks[i]):
125
                chunks[i + 1] = u"  "
5820.1.27 by INADA Naoki
Fix error when fix_sentence_endings=True.
126
                i += 2
127
            else:
128
                i += 1
129
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
130
    def _handle_long_word(self, chunks, cur_line, cur_len, width):
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
131
        # Figure out when indent is larger than the specified width, and make
132
        # sure at least one character is stripped off on every pass
133
        if width < 2:
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
134
            space_left = chunks[-1] and self._width(chunks[-1][0]) or 1
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
135
        else:
136
            space_left = width - cur_len
137
138
        # If we're allowed to break long words, then do so: put as much
139
        # of the next chunk onto the current line as will fit.
140
        if self.break_long_words:
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
141
            head, rest = self._cut(chunks[-1], space_left)
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
142
            cur_line.append(head)
5820.1.5 by INADA Naoki
Make UTextWrapper support byte string and add tests including Python's
143
            if rest:
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
144
                chunks[-1] = rest
145
            else:
146
                del chunks[-1]
147
148
        # Otherwise, we have to preserve the long word intact.  Only add
149
        # it to the current line if there's nothing already there --
150
        # that minimizes how much we violate the width constraint.
151
        elif not cur_line:
152
            cur_line.append(chunks.pop())
153
154
        # If we're not allowed to break long words, and there's already
155
        # text on the current line, do nothing.  Next time through the
156
        # main loop of _wrap_chunks(), we'll wind up here again, but
157
        # cur_len will be zero, so the next line will be entirely
158
        # devoted to the long word that we can't handle right now.
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
159
160
    def _wrap_chunks(self, chunks):
161
        lines = []
162
        if self.width <= 0:
163
            raise ValueError("invalid width %r (must be > 0)" % self.width)
7112.1.1 by Jelmer Vernooij
Fix utextwrap tests on Python 3.5.
164
        if self.max_lines is not None:
165
            if self.max_lines > 1:
166
                indent = self.subsequent_indent
167
            else:
168
                indent = self.initial_indent
169
            if self._width(indent) + self._width(self.placeholder.lstrip()) > self.width:
170
                raise ValueError("placeholder too large for max width")
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
171
172
        # Arrange in reverse order so items can be efficiently popped
173
        # from a stack of chucks.
174
        chunks.reverse()
175
176
        while chunks:
177
178
            # Start the list of chunks that will make up the current line.
179
            # cur_len is just the length of all the chunks in cur_line.
180
            cur_line = []
181
            cur_len = 0
182
183
            # Figure out which static string will prefix this line.
184
            if lines:
185
                indent = self.subsequent_indent
186
            else:
187
                indent = self.initial_indent
188
189
            # Maximum width for this line.
190
            width = self.width - len(indent)
191
192
            # First chunk on line is whitespace -- drop it, unless this
193
            # is the very beginning of the text (ie. no lines started yet).
194
            if self.drop_whitespace and chunks[-1].strip() == '' and lines:
195
                del chunks[-1]
196
197
            while chunks:
198
                # Use _width instead of len for east asian width
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
199
                l = self._width(chunks[-1])
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
200
201
                # Can at least squeeze this chunk onto the current line.
202
                if cur_len + l <= width:
203
                    cur_line.append(chunks.pop())
204
                    cur_len += l
205
206
                # Nope, this line is full.
207
                else:
208
                    break
209
210
            # The current line is full, and the next chunk is too big to
211
            # fit on *any* line (not just this one).
5820.1.19 by INADA Naoki
Add keyword parameter 'ambiguous_width' that specifies width for character
212
            if chunks and self._width(chunks[-1]) > width:
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
213
                self._handle_long_word(chunks, cur_line, cur_len, width)
7112.1.1 by Jelmer Vernooij
Fix utextwrap tests on Python 3.5.
214
                cur_len = sum(map(len, cur_line))
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
215
216
            # If the last chunk on this line is all whitespace, drop it.
5820.1.26 by INADA Naoki
Cleanup. Remove spaces in empty line and shorten line having 80 characters.
217
            if self.drop_whitespace and cur_line and not cur_line[-1].strip():
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
218
                del cur_line[-1]
219
220
            # Convert current line back to a string and store it in list
221
            # of all lines (return value).
222
            if cur_line:
7112.1.1 by Jelmer Vernooij
Fix utextwrap tests on Python 3.5.
223
                if (self.max_lines is None or
224
                    len(lines) + 1 < self.max_lines or
225
                    (not chunks or
7143.15.2 by Jelmer Vernooij
Run autopep8.
226
                        self.drop_whitespace and
7112.1.1 by Jelmer Vernooij
Fix utextwrap tests on Python 3.5.
227
                     len(chunks) == 1 and
228
                     not chunks[0].strip()) and cur_len <= width):
229
                    # Convert current line back to a string and store it in
230
                    # list of all lines (return value).
231
                    lines.append(indent + u''.join(cur_line))
232
                else:
233
                    while cur_line:
234
                        if (cur_line[-1].strip() and
7143.15.2 by Jelmer Vernooij
Run autopep8.
235
                                cur_len + self._width(self.placeholder) <= width):
7112.1.1 by Jelmer Vernooij
Fix utextwrap tests on Python 3.5.
236
                            cur_line.append(self.placeholder)
237
                            lines.append(indent + ''.join(cur_line))
238
                            break
239
                        cur_len -= self._width(cur_line[-1])
240
                        del cur_line[-1]
241
                    else:
242
                        if lines:
243
                            prev_line = lines[-1].rstrip()
244
                            if (self._width(prev_line) + self._width(self.placeholder) <=
245
                                    self.width):
246
                                lines[-1] = prev_line + self.placeholder
247
                                break
248
                        lines.append(indent + self.placeholder.lstrip())
249
                    break
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
250
251
        return lines
252
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
253
    def _split(self, text):
7012.2.1 by Jelmer Vernooij
Always pass in unicode to utextwrap.
254
        chunks = textwrap.TextWrapper._split(self, osutils.safe_unicode(text))
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
255
        cjk_split_chunks = []
256
        for chunk in chunks:
257
            prev_pos = 0
258
            for pos, char in enumerate(chunk):
5820.1.27 by INADA Naoki
Fix error when fix_sentence_endings=True.
259
                if self._unicode_char_width(char) == 2:
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
260
                    if prev_pos < pos:
261
                        cjk_split_chunks.append(chunk[prev_pos:pos])
262
                    cjk_split_chunks.append(char)
7143.15.2 by Jelmer Vernooij
Run autopep8.
263
                    prev_pos = pos + 1
5820.1.10 by INADA Naoki
utextwrap: Change a way to split between CJK characters.
264
            if prev_pos < len(chunk):
265
                cjk_split_chunks.append(chunk[prev_pos:])
266
        return cjk_split_chunks
267
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
268
    def wrap(self, text):
269
        # ensure text is unicode
7112.1.2 by Jelmer Vernooij
Fix test on python2.
270
        return textwrap.TextWrapper.wrap(self, osutils.safe_unicode(text))
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
271
272
# -- Convenience interface ---------------------------------------------
273
7143.15.2 by Jelmer Vernooij
Run autopep8.
274
5820.1.2 by INADA Naoki
bzrlib.utextwrap uses bzrlib.osutils.terminal_width() when width is not specified.
275
def wrap(text, width=None, **kwargs):
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
276
    """Wrap a single paragraph of text, returning a list of wrapped lines.
277
278
    Reformat the single paragraph in 'text' so it fits in lines of no
279
    more than 'width' columns, and return a list of wrapped lines.  By
280
    default, tabs in 'text' are expanded with string.expandtabs(), and
281
    all other whitespace characters (including newline) are converted to
282
    space.  See TextWrapper class for available keyword args to customize
283
    wrapping behaviour.
284
    """
5820.1.9 by INADA Naoki
Default width of UTextWrapper is also osutils.terminal_widtth() and
285
    return UTextWrapper(width=width, **kwargs).wrap(text)
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
286
7143.15.2 by Jelmer Vernooij
Run autopep8.
287
5820.1.2 by INADA Naoki
bzrlib.utextwrap uses bzrlib.osutils.terminal_width() when width is not specified.
288
def fill(text, width=None, **kwargs):
5820.1.1 by INADA Naoki
Add utextwrap that is same to textwrap but supports double width characters in east asia.
289
    """Fill a single paragraph of text, returning a new string.
290
291
    Reformat the single paragraph in 'text' to fit in lines of no more
292
    than 'width' columns, and return a new string containing the entire
293
    wrapped paragraph.  As with wrap(), tabs are expanded and other
294
    whitespace characters converted to space.  See TextWrapper class for
295
    available keyword args to customize wrapping behaviour.
296
    """
5820.1.9 by INADA Naoki
Default width of UTextWrapper is also osutils.terminal_widtth() and
297
    return UTextWrapper(width=width, **kwargs).fill(text)