# -*- coding: utf-8 -*-
#
# Copyright (c) 2010 Red Hat, Inc.
# Copyright (c) 2010 Ville Skyttä
# Copyright (c) 2009 Tim Lauridsen
# Copyright (c) 2007 Marcus Kuhn
#
# kitchen is free software; you can redistribute it and/or modify it under the
# terms of the GNU Lesser General Public License as published by the Free
# Software Foundation; either version 2.1 of the License, or (at your option)
# any later version.
#
# kitchen is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for
# more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with kitchen; if not, see <http://www.gnu.org/licenses/>
#
# Authors:
# James Antill <james@fedoraproject.org>
# Marcus Kuhn
# Toshio Kuratomi <toshio@fedoraproject.org>
# Tim Lauridsen
# Ville Skyttä
#
# Portions of this are from yum/i18n.py
'''
-----------------------
Format Text for Display
-----------------------
Functions related to displaying unicode text. Unicode characters don't all
have the same width so we need helper functions for displaying them.
.. versionadded:: 0.2 kitchen.display API 1.0.0
'''
import itertools
import unicodedata
from kitchen import b_
from kitchen.text.converters import to_unicode, to_bytes
from kitchen.text.exceptions import ControlCharError
# This is ported from ustr_utf8_* which I got from:
# http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
# I've tried to leave it close to the original C (same names etc.) so that
# it is easy to read/compare both versions... James Antilles
#
# Reimplemented quite a bit of this for speed. Use the bzr log or annotate
# commands to see what I've changed since importing this file.-Toshio Kuratomi
# ----------------------------- BEG utf8 ------------------to-----------
# This is an implementation of wcwidth() and wcswidth() (defined in
# IEEE Std 1002.1-2001) for Unicode.
#
# http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html
# http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html
#
# In fixed-width output devices, Latin characters all occupy a single
# "cell" position of equal width, whereas ideographic CJK characters
# occupy two such cells. Interoperability between terminal-line
# applications and (teletype-style) character terminals using the
# UTF-8 encoding requires agreement on which character should advance
# the cursor by how many cell positions. No established formal
# standards exist at present on which Unicode character shall occupy
# how many cell positions on character terminals. These routines are
# a first attempt of defining such behavior based on simple rules
# applied to data provided by the Unicode Consortium.
#
# [...]
#
# Markus Kuhn -- 2007-05-26 (Unicode 5.0)
#
# Permission to use, copy, modify, and distribute this software
# for any purpose and without fee is hereby granted. The author
# disclaims all warranties with regard to this software.
#
# Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
# Renamed but still pretty much JA's port of MK's code
def _interval_bisearch(value, table):
'''Binary search in an interval table.
:arg value: numeric value to search for
:arg table: Ordered list of intervals. This is a list of two-tuples. The
elements of the two-tuple define an interval's start and end points.
:returns: If :attr:`value` is found within an interval in the :attr:`table`
return :data:`True`. Otherwise, :data:`False`
This function checks whether a numeric value is present within a table
of intervals. It checks using a binary search algorithm, dividing the
list of values in half and checking against the values until it determines
whether the value is in the table.
'''
minimum = 0
maximum = len(table) - 1
if value < table[minimum][0] or value > table[maximum][1]:
return False
while maximum >= minimum:
mid = (minimum + maximum) / 2
if value > table[mid][1]:
minimum = mid + 1
elif value < table[mid][0]:
maximum = mid - 1
else:
return True
return False
_COMBINING = (
(0x300, 0x36f), (0x483, 0x489), (0x591, 0x5bd),
(0x5bf, 0x5bf), (0x5c1, 0x5c2), (0x5c4, 0x5c5),
(0x5c7, 0x5c7), (0x600, 0x603), (0x610, 0x61a),
(0x64b, 0x65e), (0x670, 0x670), (0x6d6, 0x6e4),
(0x6e7, 0x6e8), (0x6ea, 0x6ed), (0x70f, 0x70f),
(0x711, 0x711), (0x730, 0x74a), (0x7a6, 0x7b0),
(0x7eb, 0x7f3), (0x816, 0x819), (0x81b, 0x823),
(0x825, 0x827), (0x829, 0x82d), (0x901, 0x902),
(0x93c, 0x93c), (0x941, 0x948), (0x94d, 0x94d),
(0x951, 0x954), (0x962, 0x963), (0x981, 0x981),
(0x9bc, 0x9bc), (0x9c1, 0x9c4), (0x9cd, 0x9cd),
(0x9e2, 0x9e3), (0xa01, 0xa02), (0xa3c, 0xa3c),
(0xa41, 0xa42), (0xa47, 0xa48), (0xa4b, 0xa4d),
(0xa70, 0xa71), (0xa81, 0xa82), (0xabc, 0xabc),
(0xac1, 0xac5), (0xac7, 0xac8), (0xacd, 0xacd),
(0xae2, 0xae3), (0xb01, 0xb01), (0xb3c, 0xb3c),
(0xb3f, 0xb3f), (0xb41, 0xb43), (0xb4d, 0xb4d),
(0xb56, 0xb56), (0xb82, 0xb82), (0xbc0, 0xbc0),
(0xbcd, 0xbcd), (0xc3e, 0xc40), (0xc46, 0xc48),
(0xc4a, 0xc4d), (0xc55, 0xc56), (0xcbc, 0xcbc),
(0xcbf, 0xcbf), (0xcc6, 0xcc6), (0xccc, 0xccd),
(0xce2, 0xce3), (0xd41, 0xd43), (0xd4d, 0xd4d),
(0xdca, 0xdca), (0xdd2, 0xdd4), (0xdd6, 0xdd6),
(0xe31, 0xe31), (0xe34, 0xe3a), (0xe47, 0xe4e),
(0xeb1, 0xeb1), (0xeb4, 0xeb9), (0xebb, 0xebc),
(0xec8, 0xecd), (0xf18, 0xf19), (0xf35, 0xf35),
(0xf37, 0xf37), (0xf39, 0xf39), (0xf71, 0xf7e),
(0xf80, 0xf84), (0xf86, 0xf87), (0xf90, 0xf97),
(0xf99, 0xfbc), (0xfc6, 0xfc6), (0x102d, 0x1030),
(0x1032, 0x1032), (0x1036, 0x1037), (0x1039, 0x103a),
(0x1058, 0x1059), (0x108d, 0x108d), (0x1160, 0x11ff),
(0x135f, 0x135f), (0x1712, 0x1714), (0x1732, 0x1734),
(0x1752, 0x1753), (0x1772, 0x1773), (0x17b4, 0x17b5),
(0x17b7, 0x17bd), (0x17c6, 0x17c6), (0x17c9, 0x17d3),
(0x17dd, 0x17dd), (0x180b, 0x180d), (0x18a9, 0x18a9),
(0x1920, 0x1922), (0x1927, 0x1928), (0x1932, 0x1932),
(0x1939, 0x193b), (0x1a17, 0x1a18), (0x1a60, 0x1a60),
(0x1a75, 0x1a7c), (0x1a7f, 0x1a7f), (0x1b00, 0x1b03),
(0x1b34, 0x1b34), (0x1b36, 0x1b3a), (0x1b3c, 0x1b3c),
(0x1b42, 0x1b42), (0x1b44, 0x1b44), (0x1b6b, 0x1b73),
(0x1baa, 0x1baa), (0x1c37, 0x1c37), (0x1cd0, 0x1cd2),
(0x1cd4, 0x1ce0), (0x1ce2, 0x1ce8), (0x1ced, 0x1ced),
(0x1dc0, 0x1de6), (0x1dfd, 0x1dff), (0x200b, 0x200f),
(0x202a, 0x202e), (0x2060, 0x2063), (0x206a, 0x206f),
(0x20d0, 0x20f0), (0x2cef, 0x2cf1), (0x2de0, 0x2dff),
(0x302a, 0x302f), (0x3099, 0x309a), (0xa66f, 0xa66f),
(0xa67c, 0xa67d), (0xa6f0, 0xa6f1), (0xa806, 0xa806),
(0xa80b, 0xa80b), (0xa825, 0xa826), (0xa8c4, 0xa8c4),
(0xa8e0, 0xa8f1), (0xa92b, 0xa92d), (0xa953, 0xa953),
(0xa9b3, 0xa9b3), (0xa9c0, 0xa9c0), (0xaab0, 0xaab0),
(0xaab2, 0xaab4), (0xaab7, 0xaab8), (0xaabe, 0xaabf),
(0xaac1, 0xaac1), (0xabed, 0xabed), (0xfb1e, 0xfb1e),
(0xfe00, 0xfe0f), (0xfe20, 0xfe26), (0xfeff, 0xfeff),
(0xfff9, 0xfffb), (0x101fd, 0x101fd), (0x10a01, 0x10a03),
(0x10a05, 0x10a06), (0x10a0c, 0x10a0f), (0x10a38, 0x10a3a),
(0x10a3f, 0x10a3f), (0x110b9, 0x110ba), (0x1d165, 0x1d169),
(0x1d16d, 0x1d182), (0x1d185, 0x1d18b), (0x1d1aa, 0x1d1ad),
(0x1d242, 0x1d244), (0xe0001, 0xe0001), (0xe0020, 0xe007f),
(0xe0100, 0xe01ef), )
'''
Internal table, provided by this module to list :term:`code points` which
combine with other characters and therefore should have no :term:`textual
width`. This is a sorted :class:`tuple` of non-overlapping intervals. Each
interval is a :class:`tuple` listing a starting :term:`code point` and ending
:term:`code point`. Every :term:`code point` between the two end points is
a combining character.
.. seealso::
:func:`~kitchen.text.display._generate_combining_table`
for how this table is generated
This table was last regenerated on python-2.7.0 with
:data:`unicodedata.unidata_version` 5.1.0
'''
# New function from Toshio Kuratomi (LGPLv2+)
def _generate_combining_table():
'''Combine Markus Kuhn's data with :mod:`unicodedata` to make combining
char list
:rtype: :class:`tuple` of tuples
:returns: :class:`tuple` of intervals of :term:`code points` that are
combining character. Each interval is a 2-:class:`tuple` of the
starting :term:`code point` and the ending :term:`code point` for the
combining characters.
In normal use, this function serves to tell how we're generating the
combining char list. For speed reasons, we use this to generate a static
list and just use that later.
Markus Kuhn's list of combining characters is more complete than what's in
the python :mod:`unicodedata` library but the python :mod:`unicodedata` is
synced against later versions of the unicode database
This is used to generate the :data:`~kitchen.text.display._COMBINING`
table.
'''
# Marcus Kuhn's sorted list of non-overlapping intervals of non-spacing
# characters generated ifrom Unicode 5.0 data by:
# "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c"
markus_kuhn_combining_5_0 = (
( 0x0300, 0x036F ), ( 0x0483, 0x0486 ), ( 0x0488, 0x0489 ),
( 0x0591, 0x05BD ), ( 0x05BF, 0x05BF ), ( 0x05C1, 0x05C2 ),
( 0x05C4, 0x05C5 ), ( 0x05C7, 0x05C7 ), ( 0x0600, 0x0603 ),
( 0x0610, 0x0615 ), ( 0x064B, 0x065E ), ( 0x0670, 0x0670 ),
( 0x06D6, 0x06E4 ), ( 0x06E7, 0x06E8 ), ( 0x06EA, 0x06ED ),
( 0x070F, 0x070F ), ( 0x0711, 0x0711 ), ( 0x0730, 0x074A ),
( 0x07A6, 0x07B0 ), ( 0x07EB, 0x07F3 ), ( 0x0901, 0x0902 ),
( 0x093C, 0x093C ), ( 0x0941, 0x0948 ), ( 0x094D, 0x094D ),
( 0x0951, 0x0954 ), ( 0x0962, 0x0963 ), ( 0x0981, 0x0981 ),
( 0x09BC, 0x09BC ), ( 0x09C1, 0x09C4 ), ( 0x09CD, 0x09CD ),
( 0x09E2, 0x09E3 ), ( 0x0A01, 0x0A02 ), ( 0x0A3C, 0x0A3C ),
( 0x0A41, 0x0A42 ), ( 0x0A47, 0x0A48 ), ( 0x0A4B, 0x0A4D ),
( 0x0A70, 0x0A71 ), ( 0x0A81, 0x0A82 ), ( 0x0ABC, 0x0ABC ),
( 0x0AC1, 0x0AC5 ), ( 0x0AC7, 0x0AC8 ), ( 0x0ACD, 0x0ACD ),
( 0x0AE2, 0x0AE3 ), ( 0x0B01, 0x0B01 ), ( 0x0B3C, 0x0B3C ),
( 0x0B3F, 0x0B3F ), ( 0x0B41, 0x0B43 ), ( 0x0B4D, 0x0B4D ),
( 0x0B56, 0x0B56 ), ( 0x0B82, 0x0B82 ), ( 0x0BC0, 0x0BC0 ),
( 0x0BCD, 0x0BCD ), ( 0x0C3E, 0x0C40 ), ( 0x0C46, 0x0C48 ),
( 0x0C4A, 0x0C4D ), ( 0x0C55, 0x0C56 ), ( 0x0CBC, 0x0CBC ),
( 0x0CBF, 0x0CBF ), ( 0x0CC6, 0x0CC6 ), ( 0x0CCC, 0x0CCD ),
( 0x0CE2, 0x0CE3 ), ( 0x0D41, 0x0D43 ), ( 0x0D4D, 0x0D4D ),
( 0x0DCA, 0x0DCA ), ( 0x0DD2, 0x0DD4 ), ( 0x0DD6, 0x0DD6 ),
( 0x0E31, 0x0E31 ), ( 0x0E34, 0x0E3A ), ( 0x0E47, 0x0E4E ),
( 0x0EB1, 0x0EB1 ), ( 0x0EB4, 0x0EB9 ), ( 0x0EBB, 0x0EBC ),
( 0x0EC8, 0x0ECD ), ( 0x0F18, 0x0F19 ), ( 0x0F35, 0x0F35 ),
( 0x0F37, 0x0F37 ), ( 0x0F39, 0x0F39 ), ( 0x0F71, 0x0F7E ),
( 0x0F80, 0x0F84 ), ( 0x0F86, 0x0F87 ), ( 0x0F90, 0x0F97 ),
( 0x0F99, 0x0FBC ), ( 0x0FC6, 0x0FC6 ), ( 0x102D, 0x1030 ),
( 0x1032, 0x1032 ), ( 0x1036, 0x1037 ), ( 0x1039, 0x1039 ),
( 0x1058, 0x1059 ), ( 0x1160, 0x11FF ), ( 0x135F, 0x135F ),
( 0x1712, 0x1714 ), ( 0x1732, 0x1734 ), ( 0x1752, 0x1753 ),
( 0x1772, 0x1773 ), ( 0x17B4, 0x17B5 ), ( 0x17B7, 0x17BD ),
( 0x17C6, 0x17C6 ), ( 0x17C9, 0x17D3 ), ( 0x17DD, 0x17DD ),
( 0x180B, 0x180D ), ( 0x18A9, 0x18A9 ), ( 0x1920, 0x1922 ),
( 0x1927, 0x1928 ), ( 0x1932, 0x1932 ), ( 0x1939, 0x193B ),
( 0x1A17, 0x1A18 ), ( 0x1B00, 0x1B03 ), ( 0x1B34, 0x1B34 ),
( 0x1B36, 0x1B3A ), ( 0x1B3C, 0x1B3C ), ( 0x1B42, 0x1B42 ),
( 0x1B6B, 0x1B73 ), ( 0x1DC0, 0x1DCA ), ( 0x1DFE, 0x1DFF ),
( 0x200B, 0x200F ), ( 0x202A, 0x202E ), ( 0x2060, 0x2063 ),
( 0x206A, 0x206F ), ( 0x20D0, 0x20EF ), ( 0x302A, 0x302F ),
( 0x3099, 0x309A ), ( 0xA806, 0xA806 ), ( 0xA80B, 0xA80B ),
( 0xA825, 0xA826 ), ( 0xFB1E, 0xFB1E ), ( 0xFE00, 0xFE0F ),
( 0xFE20, 0xFE23 ), ( 0xFEFF, 0xFEFF ), ( 0xFFF9, 0xFFFB ),
( 0x10A01, 0x10A03 ), ( 0x10A05, 0x10A06 ), ( 0x10A0C, 0x10A0F ),
( 0x10A38, 0x10A3A ), ( 0x10A3F, 0x10A3F ), ( 0x1D167, 0x1D169 ),
( 0x1D173, 0x1D182 ), ( 0x1D185, 0x1D18B ), ( 0x1D1AA, 0x1D1AD ),
( 0x1D242, 0x1D244 ), ( 0xE0001, 0xE0001 ), ( 0xE0020, 0xE007F ),
( 0xE0100, 0xE01EF ))
combining = []
in_interval = False
interval = []
for codepoint in xrange (0, 0xFFFFF + 1):
if _interval_bisearch(codepoint, markus_kuhn_combining_5_0) or \
unicodedata.combining(unichr(codepoint)):
if not in_interval:
# Found first part of an interval
interval = [codepoint]
in_interval = True
else:
if in_interval:
in_interval = False
interval.append(codepoint - 1)
combining.append(interval)
if in_interval:
# If we're at the end and the interval is open, close it.
# :W0631: We looped through a static range so we know codepoint is
# defined here
#pylint:disable-msg=W0631
interval.append(codepoint)
combining.append(interval)
return tuple(itertools.imap(tuple, combining))
# New function from Toshio Kuratomi (LGPLv2+)
def _print_combining_table():
'''Print out a new :data:`_COMBINING` table
This will print a new :data:`_COMBINING` table in the format used in
:file:`kitchen/text/display.py`. It's useful for updating the
:data:`_COMBINING` table with updated data from a new python as the format
won't change from what's already in the file.
'''
table = _generate_combining_table()
entries = 0
print '_COMBINING = ('
for pair in table:
if entries >= 3:
entries = 0
print
if entries == 0:
print ' ',
entries += 1
entry = '(0x%x, 0x%x),' % pair
print entry,
print ')'
# Handling of control chars rewritten. Rest is JA's port of MK's C code.
# -Toshio Kuratomi
def _ucp_width(ucs, control_chars='guess'):
'''Get the :term:`textual width` of a ucs character
:arg ucs: integer representing a single unicode :term:`code point`
:kwarg control_chars: specify how to deal with :term:`control characters`.
Possible values are:
:guess: (default) will take a guess for :term:`control character`
widths. Most codes will return zero width. ``backspace``,
``delete``, and ``clear delete`` return -1. ``escape`` currently
returns -1 as well but this is not guaranteed as it's not always
correct
:strict: will raise :exc:`~kitchen.text.exceptions.ControlCharError`
if a :term:`control character` is encountered
:raises ControlCharError: if the :term:`code point` is a unicode
:term:`control character` and :attr:`control_chars` is set to 'strict'
:returns: :term:`textual width` of the character.
.. note::
It's important to remember this is :term:`textual width` and not the
number of characters or bytes.
'''
# test for 8-bit control characters
if ucs < 32 or (ucs < 0xa0 and ucs >= 0x7f):
# Control character detected
if control_chars == 'strict':
raise ControlCharError(b_('_ucp_width does not understand how to'
' assign a width value to control characters.'))
if ucs in (0x08, 0x07F, 0x94):
# Backspace, delete, and clear delete remove a single character
return -1
if ucs == 0x1b:
# Excape is tricky. It removes some number of characters that
# come after it but the amount is dependent on what is
# interpreting the code.
# So this is going to often be wrong but other values will be
# wrong as well.
return -1
# All other control characters get 0 width
return 0
if _interval_bisearch(ucs, _COMBINING):
# Combining characters return 0 width as they will be combined with
# the width from other characters
return 0
# if we arrive here, ucs is not a combining or C0/C1 control character
return (1 +
(ucs >= 0x1100 and
(ucs <= 0x115f or # Hangul Jamo init. consonants
ucs == 0x2329 or ucs == 0x232a or
(ucs >= 0x2e80 and ucs <= 0xa4cf and
ucs != 0x303f) or # CJK ... Yi
(ucs >= 0xac00 and ucs <= 0xd7a3) or # Hangul Syllables
(ucs >= 0xf900 and ucs <= 0xfaff) or # CJK Compatibility Ideographs
(ucs >= 0xfe10 and ucs <= 0xfe19) or # Vertical forms
(ucs >= 0xfe30 and ucs <= 0xfe6f) or # CJK Compatibility Forms
(ucs >= 0xff00 and ucs <= 0xff60) or # Fullwidth Forms
(ucs >= 0xffe0 and ucs <= 0xffe6) or
(ucs >= 0x20000 and ucs <= 0x2fffd) or
(ucs >= 0x30000 and ucs <= 0x3fffd))))
# Wholly rewritten by me (LGPLv2+) -Toshio Kuratomi
def textual_width(msg, control_chars='guess', encoding='utf-8',
errors='replace'):
'''Get the :term:`textual width` of a string
:arg msg: :class:`unicode` string or byte :class:`str` to get the width of
:kwarg control_chars: specify how to deal with :term:`control characters`.
Possible values are:
:guess: (default) will take a guess for :term:`control character`
widths. Most codes will return zero width. ``backspace``,
``delete``, and ``clear delete`` return -1. ``escape`` currently
returns -1 as well but this is not guaranteed as it's not always
correct
:strict: will raise :exc:`kitchen.text.exceptions.ControlCharError`
if a :term:`control character` is encountered
:kwarg encoding: If we are given a byte :class:`str` this is used to
decode it into :class:`unicode` string. Any characters that are not
decodable in this encoding will get a value dependent on the
:attr:`errors` parameter.
:kwarg errors: How to treat errors encoding the byte :class:`str` to
:class:`unicode` string. Legal values are the same as for
:func:`kitchen.text.converters.to_unicode`. The default value of
``replace`` will cause undecodable byte sequences to have a width of
one. ``ignore`` will have a width of zero.
:raises ControlCharError: if :attr:`msg` contains a :term:`control
character` and :attr:`control_chars` is ``strict``.
:returns: :term:`Textual width` of the :attr:`msg`. This is the amount of
space that the string will consume on a monospace display. It's
measured in the number of cell positions or columns it will take up on
a monospace display. This is **not** the number of glyphs that are in
the string.
.. note::
This function can be wrong sometimes because Unicode does not specify
a strict width value for all of the :term:`code points`. In
particular, we've found that some Tamil characters take up to four
character cells but we return a lesser amount.
'''
# On python 2.6.4, x86_64, I've benchmarked a few alternate
# implementations::
#
# timeit.repeat('display.textual_width(data)',
# 'from __main__ import display, data', number=100)
# I varied data by size and content (1MB of ascii, a few words, 43K utf8,
# unicode type
#
# :this implementation: fastest across the board
#
# :list comprehension: 6-16% slower
# return sum([_ucp_width(ord(c), control_chars=control_chars)
# for c in msg])
#
# :generator expression: 9-18% slower
# return sum((_ucp_width(ord(c), control_chars=control_chars) for c in
# msg))
#
# :lambda: 10-19% slower
# return sum(itertools.imap(lambda x: _ucp_width(ord(x), control_chars),
# msg))
#
# :partial application: 13-22% slower
# func = functools.partial(_ucp_width, control_chars=control_chars)
# return sum(itertools.imap(func, itertools.imap(ord, msg)))
#
# :the original code: 4-38% slower
# The 4% was for the short, ascii only string. All the other pieces of
# data yielded over 30% slower times.
# Non decodable data is just assigned a single cell width
msg = to_unicode(msg, encoding=encoding, errors=errors)
# Add the width of each char
return sum(
# calculate width of each char
itertools.starmap(_ucp_width,
# Setup the arguments to _ucp_width
itertools.izip(
# int value of each char
itertools.imap(ord, msg),
# control_chars arg in a form that izip will deal with
itertools.repeat(control_chars))))
# Wholly rewritten by me -Toshio Kuratomi
def textual_width_chop(msg, chop, encoding='utf-8', errors='replace'):
'''Given a string, return it chopped to a given :term:`textual width`
:arg msg: :class:`unicode` string or byte :class:`str` to chop
:arg chop: Chop :attr:`msg` if it exceeds this :term:`textual width`
:kwarg encoding: If we are given a byte :class:`str`, this is used to
decode it into a :class:`unicode` string. Any characters that are not
decodable in this encoding will be assigned a width of one.
:kwarg errors: How to treat errors encoding the byte :class:`str` to
:class:`unicode`. Legal values are the same as for
:func:`kitchen.text.converters.to_unicode`
:rtype: :class:`unicode` string
:returns: :class:`unicode` string of the :attr:`msg` chopped at the given
:term:`textual width`
This is what you want to use instead of ``%.*s``, as it does the "right"
thing with regard to :term:`UTF-8` sequences, :term:`control characters`,
and characters that take more than one cell position. Eg::
>>> # Wrong: only displays 8 characters because it is operating on bytes
>>> print "%.*s" % (10, 'café ñunru!')
café ñun
>>> # Properly operates on graphemes
>>> '%s' % (textual_width_chop('café ñunru!', 10))
café ñunru
>>> # takes too many columns because the kanji need two cell positions
>>> print '1234567890\\n%.*s' % (10, u'一二三四五六七八九十')
1234567890
一二三四五六七八九十
>>> # Properly chops at 10 columns
>>> print '1234567890\\n%s' % (textual_width_chop(u'一二三四五六七八九十', 10))
1234567890
一二三四五
'''
msg = to_unicode(msg, encoding=encoding, errors=errors)
width = textual_width(msg)
if width <= chop:
return msg
maximum = len(msg)
if maximum > chop * 2:
# A character can take at most 2 cell positions so this is the actual
# maximum
maximum = chop * 2
minimum = 0
eos = maximum
if eos > chop:
eos = chop
width = textual_width(msg[:eos])
while True:
# if current width is high,
if width > chop:
# calculate new midpoint
mid = minimum + (eos - minimum) / 2
if mid == eos:
break
if (eos - chop) < (eos - mid):
while width > chop:
width = width - _ucp_width(ord(msg[eos-1]))
eos -= 1
return msg[:eos]
# subtract distance between eos and mid from width
width = width - textual_width(msg[mid:eos])
maximum = eos
eos = mid
# if current width is low,
elif width < chop:
# Note: at present, the if (eos - chop) < (eos - mid):
# short-circuit above means that we never use this branch.
# calculate new midpoint
mid = eos + (maximum - eos) / 2
if mid == eos:
break
if (chop - eos) < (mid - eos):
while width < chop:
new_width = _ucp_width(ord(msg[eos]))
width = width + new_width
eos += 1
return msg[:eos]
# add distance between eos and new mid to width
width = width + textual_width(msg[eos:mid])
minimum = eos
eos = mid
if eos > maximum:
eos = maximum
break
# if current is just right
else:
return msg[:eos]
return msg[:eos]
# I made some adjustments for using unicode but largely unchanged from JA's
# port of MK's code -Toshio
def textual_width_fill(msg, fill, chop=None, left=True, prefix='', suffix=''):
'''Expand a :class:`unicode` string to a specified :term:`textual width`
or chop to same
:arg msg: :class:`unicode` string to format
:arg fill: pad string until the :term:`textual width` of the string is
this length
:kwarg chop: before doing anything else, chop the string to this length.
Default: Don't chop the string at all
:kwarg left: If :data:`True` (default) left justify the string and put the
padding on the right. If :data:`False`, pad on the left side.
:kwarg prefix: Attach this string before the field we're filling
:kwarg suffix: Append this string to the end of the field we're filling
:rtype: :class:`unicode` string
:returns: :attr:`msg` formatted to fill the specified width. If no
:attr:`chop` is specified, the string could exceed the fill length
when completed. If :attr:`prefix` or :attr:`suffix` are printable
characters, the string could be longer than the fill width.
.. note::
:attr:`prefix` and :attr:`suffix` should be used for "invisible"
characters like highlighting, color changing escape codes, etc. The
fill characters are appended outside of any :attr:`prefix` or
:attr:`suffix` elements. This allows you to only highlight
:attr:`msg` inside of the field you're filling.
.. warning::
:attr:`msg`, :attr:`prefix`, and :attr:`suffix` should all be
representable as unicode characters. In particular, any escape
sequences in :attr:`prefix` and :attr:`suffix` need to be convertible
to :class:`unicode`. If you need to use byte sequences here rather
than unicode characters, use
:func:`~kitchen.text.display.byte_string_textual_width_fill` instead.
This function expands a string to fill a field of a particular
:term:`textual width`. Use it instead of ``%*.*s``, as it does the
"right" thing with regard to :term:`UTF-8` sequences, :term:`control
characters`, and characters that take more than one cell position in
a display. Example usage::
>>> msg = u'一二三四五六七八九十'
>>> # Wrong: This uses 10 characters instead of 10 cells:
>>> u":%-*.*s:" % (10, 10, msg[:9])
:一二三四五六七八九 :
>>> # This uses 10 cells like we really want:
>>> u":%s:" % (textual_width_fill(msg[:9], 10, 10))
:一二三四五:
>>> # Wrong: Right aligned in the field, but too many cells
>>> u"%20.10s" % (msg)
一二三四五六七八九十
>>> # Correct: Right aligned with proper number of cells
>>> u"%s" % (textual_width_fill(msg, 20, 10, left=False))
一二三四五
>>> # Wrong: Adding some escape characters to highlight the line but too many cells
>>> u"%s%20.10s%s" % (prefix, msg, suffix)
u'\x1b[7m 一二三四五六七八九十\x1b[0m'
>>> # Correct highlight of the line
>>> u"%s%s%s" % (prefix, display.textual_width_fill(msg, 20, 10, left=False), suffix)
u'\x1b[7m 一二三四五\x1b[0m'
>>> # Correct way to not highlight the fill
>>> u"%s" % (display.textual_width_fill(msg, 20, 10, left=False, prefix=prefix, suffix=suffix))
u' \x1b[7m一二三四五\x1b[0m'
'''
msg = to_unicode(msg)
if chop is not None:
msg = textual_width_chop(msg, chop)
width = textual_width(msg)
if width >= fill:
if prefix or suffix:
msg = u''.join([prefix, msg, suffix])
else:
extra = u' ' * (fill - width)
if left:
msg = u''.join([prefix, msg, suffix, extra])
else:
msg = u''.join([extra, prefix, msg, suffix])
return msg
def _textual_width_le(width, *args):
'''Optimize the common case when deciding which :term:`textual width` is
larger
:arg width: :term:`textual width` to compare against.
:arg \*args: :class:`unicode` strings to check the total :term:`textual
width` of
:returns: :data:`True` if the total length of :attr:`args` are less than
or equal to :attr:`width`. Otherwise :data:`False`.
We often want to know "does X fit in Y". It takes a while to use
:func:`textual_width` to calculate this. However, we know that the number
of canonically composed :class:`unicode` characters is always going to
have 1 or 2 for the :term:`textual width` per character. With this we can
take the following shortcuts:
1) If the number of canonically composed characters is more than width,
the true :term:`textual width` cannot be less than width.
2) If the number of canonically composed characters * 2 is less than the
width then the :term:`textual width` must be ok.
:term:`textual width` of a canonically composed :class:`unicode` string
will always be greater than or equal to the the number of :class:`unicode`
characters. So we can first check if the number of composed
:class:`unicode` characters is less than the asked for width. If it is we
can return :data:`True` immediately. If not, then we must do a full
:term:`textual width` lookup.
'''
string = ''.join(args)
string = unicodedata.normalize('NFC', string)
if len(string) > width:
return False
elif len(string) * 2 <= width:
return True
elif len(to_bytes(string)) <= width:
# Check against bytes.
# utf8 has the property of having the same amount or more bytes per
# character than textual width.
return True
else:
true_width = textual_width(string)
return true_width <= width
def wrap(text, width=70, initial_indent=u'', subsequent_indent=u'',
encoding='utf-8', errors='replace'):
'''Works like we want :func:`textwrap.wrap` to work,
:arg text: :class:`unicode` string or byte :class:`str` to wrap
:kwarg width: :term:`textual width` at which to wrap. Default: 70
:kwarg initial_indent: string to use to indent the first line. Default:
do not indent.
:kwarg subsequent_indent: string to use to wrap subsequent lines.
Default: do not indent
:kwarg encoding: Encoding to use if :attr:`text` is a byte :class:`str`
:kwarg errors: error handler to use if :attr:`text` is a byte :class:`str`
and contains some undecodable characters.
:rtype: :class:`list` of :class:`unicode` strings
:returns: list of lines that have been text wrapped and indented.
:func:`textwrap.wrap` from the |stdlib|_ has two drawbacks that this
attempts to fix:
1. It does not handle :term:`textual width`. It only operates on bytes or
characters which are both inadequate (due to multi-byte and double
width characters).
2. It malforms lists and blocks.
'''
# Tested with:
# yum info robodoc gpicview php-pear-Net-Socket wmctrl ustr moreutils
# mediawiki-HNP ocspd insight yum mousepad
# ...at 120, 80 and 40 chars.
# Also, notable among lots of others, searching for "\n ":
# exim-clamav, jpackage-utils, tcldom, synaptics, "quake3",
# perl-Class-Container, ez-ipupdate, perl-Net-XMPP, "kipi-plugins",
# perl-Apache-DBI, netcdf, python-configobj, "translate-toolkit", alpine,
# "udunits", "conntrack-tools"
#
# Note that, we "fail" on:
# alsa-plugins-jack, setools*, dblatex, uisp, "perl-Getopt-GUI-Long",
# suitesparse, "synce-serial", writer2latex, xenwatch, ltsp-utils
def _indent_at_beg(line):
'''Return the indent to use for this and (possibly) subsequent lines
:arg line: :class:`unicode` line of text to process
:rtype: tuple
:returns: tuple of count of whitespace before getting to the start of
this line followed by a count to the following indent if this
block of text is an entry in a list.
'''
# Find the first non-whitespace character
try:
char = line.strip()[0]
except IndexError:
# All whitespace
return 0, 0
else:
count = line.find(char)
# if we have a bullet character, check for list
if char not in u'-*.o\u2022\u2023\u2218':
# No bullet; not a list
return count, 0
# List: Keep searching until we hit the innermost list
nxt = _indent_at_beg(line[count+1:])
nxt = nxt[1] or nxt[0]
if nxt:
return count, count + 1 + nxt
return count, 0
initial_indent = to_unicode(initial_indent, encoding=encoding,
errors=errors)
subsequent_indent = to_unicode(subsequent_indent, encoding=encoding,
errors=errors)
subsequent_indent_width = textual_width(subsequent_indent)
text = to_unicode(text, encoding=encoding, errors=errors).rstrip(u'\n')
lines = text.expandtabs().split(u'\n')
ret = []
indent = initial_indent
wrap_last = False
cur_sab = 0
cur_spc_indent = 0
for line in lines:
line = line.rstrip(u' ')
(last_sab, last_spc_indent) = (cur_sab, cur_spc_indent)
(cur_sab, cur_spc_indent) = _indent_at_beg(line)
force_nl = False # We want to stop wrapping under "certain" conditions:
if wrap_last and cur_spc_indent: # if line starts a list or
force_nl = True
if wrap_last and cur_sab == len(line):# is empty line
force_nl = True
if wrap_last and not last_spc_indent: # if we don't continue a list
if cur_sab >= 4 and cur_sab != last_sab: # and is "block indented"
force_nl = True
if force_nl:
ret.append(indent.rstrip(u' '))
indent = subsequent_indent
wrap_last = False
if cur_sab == len(line): # empty line, remove spaces to make it easier.
line = u''
if wrap_last:
line = line.lstrip(u' ')
cur_spc_indent = last_spc_indent
if _textual_width_le(width, indent, line):
wrap_last = False
ret.append(indent + line)
indent = subsequent_indent
continue
wrap_last = True
words = line.split(u' ')
line = indent
spcs = cur_spc_indent
if not spcs and cur_sab >= 4:
spcs = cur_sab
for word in words:
if (not _textual_width_le(width, line, word) and
textual_width(line) > subsequent_indent_width):
ret.append(line.rstrip(u' '))
line = subsequent_indent + u' ' * spcs
line += word
line += u' '
indent = line.rstrip(u' ') + u' '
if wrap_last:
ret.append(indent.rstrip(u' '))
return ret
def fill(text, *args, **kwargs):
'''Works like we want :func:`textwrap.fill` to work
:arg text: :class:`unicode` string or byte :class:`str` to process
:returns: :class:`unicode` string with each line separated by a newline
.. seealso::
:func:`kitchen.text.display.wrap`
for other parameters that you can give this command.
This function is a light wrapper around :func:`kitchen.text.display.wrap`.
Where that function returns a :class:`list` of lines, this function
returns one string with each line separated by a newline.
'''
return u'\n'.join(wrap(text, *args, **kwargs))
#
# Byte strings
#
def byte_string_textual_width_fill(msg, fill, chop=None, left=True, prefix='',
suffix='', encoding='utf-8', errors='replace'):
'''Expand a byte :class:`str` to a specified :term:`textual width` or chop
to same
:arg msg: byte :class:`str` encoded in :term:`UTF-8` that we want formatted
:arg fill: pad :attr:`msg` until the :term:`textual width` is this long
:kwarg chop: before doing anything else, chop the string to this length.
Default: Don't chop the string at all
:kwarg left: If :data:`True` (default) left justify the string and put the
padding on the right. If :data:`False`, pad on the left side.
:kwarg prefix: Attach this byte :class:`str` before the field we're
filling
:kwarg suffix: Append this byte :class:`str` to the end of the field we're
filling
:rtype: byte :class:`str`
:returns: :attr:`msg` formatted to fill the specified :term:`textual
width`. If no :attr:`chop` is specified, the string could exceed the
fill length when completed. If :attr:`prefix` or :attr:`suffix` are
printable characters, the string could be longer than fill width.
.. note::
:attr:`prefix` and :attr:`suffix` should be used for "invisible"
characters like highlighting, color changing escape codes, etc. The
fill characters are appended outside of any :attr:`prefix` or
:attr:`suffix` elements. This allows you to only highlight
:attr:`msg` inside of the field you're filling.
.. seealso::
:func:`~kitchen.text.display.textual_width_fill`
For example usage. This function has only two differences.
1. it takes byte :class:`str` for :attr:`prefix` and
:attr:`suffix` so you can pass in arbitrary sequences of
bytes, not just unicode characters.
2. it returns a byte :class:`str` instead of a :class:`unicode`
string.
'''
prefix = to_bytes(prefix, encoding=encoding, errors=errors)
suffix = to_bytes(suffix, encoding=encoding, errors=errors)
if chop is not None:
msg = textual_width_chop(msg, chop, encoding=encoding, errors=errors)
width = textual_width(msg)
msg = to_bytes(msg)
if width >= fill:
if prefix or suffix:
msg = ''.join([prefix, msg, suffix])
else:
extra = ' ' * (fill - width)
if left:
msg = ''.join([prefix, msg, suffix, extra])
else:
msg = ''.join([extra, prefix, msg, suffix])
return msg
__all__ = ('byte_string_textual_width_fill', 'fill', 'textual_width',
'textual_width_chop', 'textual_width_fill', 'wrap')