added = set()
retlist = []
- for code in locales:
- if not code:
+ for locale_ in locales:
+ if not locale_:
continue
if locale is not None:
- code = locale.normalize(code)
+ locale_ = locale.normalize(locale_)
# Strip off encoding if present.
- code = code.split(".")[0]
- if code.lower() not in added:
- retlist.append(code)
- added.add(code.lower())
+ locale_ = locale_.split(".")[0]
+ if locale_.lower() not in added:
+ retlist.append(locale_)
+ added.add(locale_.lower())
# Strip off territory if present.
- code = code.split("_")[0]
- if code.lower() not in added:
- retlist.append(code)
- added.add(code.lower())
+ locale_ = locale_.split("_")[0]
+ if locale_.lower() not in added:
+ retlist.append(locale_)
+ added.add(locale_.lower())
return retlist
+"""String utility functions for collation."""
+
+__all__ = ["sortemes", "numeric", "normalize_number"]
+
import unicodedata
CONTINUE_ON = frozenset([
BREAKER = u"\u2029" # Paragraph break character
INFINITY = float('inf')
+KEEP_IN_NUMBERS = u"'.,"
+ALLOWED_IN_NUMBERS = KEEP_IN_NUMBERS + u"_"
+
+def stripends(word):
+ """Strip punctuation and symbols from the ends of a string."""
+ while word and unicodedata.category(word[0])[0] in "PS":
+ word = word[1:]
+ while word and unicodedata.category(word[-1])[0] in "PS":
+ word = word[:-1]
+ return word
+
def sortemes(string, key=lambda s: s):
"""Generate a list of sortemes for the string.
There is no formal specification for sortemes; the goal of this
function is to provide good output for Collator.sortemekey.
+
"""
words = []
categories = map(unicodedata.category, string)
previous = UNKNOWN
- def stripends(word):
- while word and unicodedata.category(word[0])[0] in "PS":
- word = word[1:]
- while word and unicodedata.category(word[-1])[0] in "PS":
- word = word[:-1]
- return word
-
def aletters(letters):
+ """Add a group of letters to the word list."""
words.append((INFINITY, stripends(letters)))
def adigits(digits):
+ """Add a group of digits to the word list."""
words.append((numeric(digits), u''))
# TODO(jfw): This kind of evolved over time, there's probably a much
# faster / more concise way to express it now.
- for i, (c, category) in enumerate(zip(string, categories)):
+ for i, (uchar, category) in enumerate(zip(string, categories)):
if letters and previous == LETTER and words:
word = stripends(words.pop()[1].strip()) + BREAKER
# Split at the first letter following a number or
# non-continuing character.
if category[0] == "L":
- letters.append(c)
+ letters.append(uchar)
if digits:
adigits(u"".join(digits).strip())
digits = []
# Split at the first number following a non-number or
# non-continuing character.
elif category[0] == "N":
- digits.append(c)
+ digits.append(uchar)
if letters:
aletters(u"".join(letters))
letters = []
previous = LETTER
# Only certain punctuation allowed in numbers.
- elif digits and c not in "',._":
+ elif digits and uchar not in ALLOWED_IN_NUMBERS:
adigits(u"".join(digits))
digits = []
previous = NUMBER
else:
if digits:
- digits.append(c)
+ digits.append(uchar)
elif letters:
- letters.append(c)
+ letters.append(uchar)
if letters and previous == LETTER and words:
word = stripends(words.pop()[1].strip()) + BREAKER
return [(i, key(w) if w else u'') for i, w in words]
def numeric(orig, invalid=INFINITY):
+ """Parse a number out of a string.
+
+ This function parses a unicode number out of the start of a
+ string. If a number cannot be found at the start, the 'invalid'
+ argument is returned.
+
+ """
+
if not orig:
return invalid
string = unicode(orig)
- for c in string:
- if c.isnumeric():
+ for uchar in string:
+ if uchar.isnumeric():
break
else:
return invalid
string = string[1:]
if not string[:1].isnumeric():
- return (invalid, orig)
+ return invalid
- string = normalize_punc(string)
+ string = normalize_number(string)
- # Otherwise we need to do this the hard way.
def _numeric(string):
+ """Interpreter a number as base 10."""
total = 0
- for c in string:
- v = unicodedata.numeric(c)
- if v >= 1 or v == 0:
+ for uchar in string:
+ number = unicodedata.numeric(uchar)
+ if number >= 1 or number == 0:
total *= 10
- total += v
+ total += number
return total
try:
except ValueError:
return mult * _numeric(string)
-def normalize_punc(string):
- string = unicode(string.strip(u",.'"))
- string = filter(lambda u: u.isnumeric() or u in u",.'", string)
+def normalize_number(string):
+ """Normalize punctuation in a number.
+
+ This function attempts to guess which characters in a number
+ represent grouping separators and which represent decimal
+ points. It returns a string that is valid to pass to Python's
+ float() routine (potentially, NaN, if nothing like a number is
+ found).
+
+ """
+
+ string = unicode(string)
+ string = filter(lambda u: u.isnumeric() or u in KEEP_IN_NUMBERS, string)
+ string = string.strip(KEEP_IN_NUMBERS)
+
commas = string.count(u",")
stops = string.count(u".")
quotes = string.count(u"'")
quotes = 0
def normalize_two(a, b, string):
- # One of each - assume the first is grouping, second is point.
+ """One of each - assume the first is grouping, second is point."""
a_idx = string.rindex(a)
b_idx = string.rindex(b)
if a_idx > b_idx:
# Single stop, but no decimal - probably grouping.
string = string.replace(u".", u"")
- return string
+ return string or "NaN"
libraries = ['icui18n', 'icuuc', 'icudata']
setup(name='collate',
- version='0',
+ version='0.1',
author="Joe Wreschnig",
author_email="joe.wreschnig@gmail.com",
description="Python text collation",
libraries=libraries)],
cmdclass=dict(build_ext=build_ext),
packages=["collate", "collate.icu"],
+ long_description="""\
+This module provides tools to sort strings in a 'human-expected'
+order. Because human expectations are fuzzy and often
+self-contradictory, the sort order is not guaranteed to be stable
+between versions of this module (rather the opposite - the primary
+reason to update it will probably be changed sort results). If
+available, this module uses the ICU localization library. Otherwise,
+it uses the system's locale database (and produces significantly worse
+results).
+"""
)
projects / python-collate.git / commitdiff