X-Git-Url: https://git.korewanetadesu.com/?p=python-collate.git;a=blobdiff_plain;f=collate%2Fstrings.py;h=8d6af993cd72d5f0fcb4fc04eb6e1953f72904c9;hp=267c6e573fc23acbb52425331708dce4b2c4c426;hb=e8b221311bf60883d3f14083bab63fd947d34509;hpb=b0490f43bebb2aca90496fcdb5b65fa89642b409 diff --git a/collate/strings.py b/collate/strings.py index 267c6e5..8d6af99 100644 --- a/collate/strings.py +++ b/collate/strings.py @@ -1,3 +1,7 @@ +"""String utility functions for collation.""" + +__all__ = ["sortemes", "numeric", "normalize_number"] + import unicodedata CONTINUE_ON = frozenset([ @@ -10,105 +14,142 @@ CONTINUE_ON = frozenset([ UNKNOWN, LETTER, NUMBER = range(3) -BREAKER = u"\u2029" +BREAKER = u"\u2029" # Paragraph break character +INFINITY = float('inf') + +KEEP_IN_NUMBERS = u"'.," +ALLOWED_IN_NUMBERS = KEEP_IN_NUMBERS + u"_" -def sortemes(string): +def stripends(word): + """Strip punctuation and symbols from the ends of a string.""" + while word and unicodedata.category(word[0])[0] in "PS": + word = word[1:] + while word and unicodedata.category(word[-1])[0] in "PS": + word = word[:-1] + return word + +def sortemes(string, key=lambda s: s): """Generate a list of sortemes for the string. A sorteme, by analogy with grapheme/morpheme/etc. is an atom of sort information. This is larger than a word boundry but smaller than a sentence boundry; roughly, a sorteme boundry occurs between - letters and numbers, between numbers and numbrs if 'too much' + letters and numbers, between numbers and numbers if 'too much' punctuation exists in between, between lines. There is no formal specification for sortemes; the goal of this function is to provide good output for Collator.sortemekey. + """ words = [] + letters = [] + digits = [] if not string: return words string = unicode(string) - start = None - last = None - mode = UNKNOWN - previous_mode = UNKNOWN - category = "XX" + categories = map(unicodedata.category, string) + previous = UNKNOWN + + def aletters(letters): + """Add a group of letters to the word list.""" + words.append((INFINITY, stripends(letters))) + def adigits(digits): + """Add a group of digits to the word list.""" + words.append((numeric(digits), u'')) # TODO(jfw): This kind of evolved over time, there's probably a much # faster / more concise way to express it now. - for i, c in enumerate(string): - broke = False - prev_category = category - this_mode = mode - category = unicodedata.category(c) + for i, (uchar, category) in enumerate(zip(string, categories)): + + if letters and previous == LETTER and words: + word = stripends(words.pop()[1].strip()) + BREAKER + letters.insert(0, word) + previous = UNKNOWN # Split at the first letter following a number or # non-continuing character. if category[0] == "L": - if mode != LETTER: - broke = True - mode = LETTER + letters.append(uchar) + if digits: + adigits(u"".join(digits).strip()) + digits = [] + previous = NUMBER # Split at the first number following a non-number or # non-continuing character. elif category[0] == "N": - if mode != NUMBER: - broke = True - mode = NUMBER - - # Split if we find a non-continuing character ("weird" ones). - elif category not in CONTINUE_ON: - broke = True - mode = UNKNOWN + digits.append(uchar) + if letters: + aletters(u"".join(letters)) + letters = [] + previous = LETTER # Only certain punctuation allowed in numbers. - elif mode == NUMBER and category[0] == "P" and c not in "',._": - broke = True - mode = UNKNOWN + elif digits and uchar not in ALLOWED_IN_NUMBERS: + adigits(u"".join(digits)) + digits = [] + previous = NUMBER + + # Split if we find a non-continuing character ("weird" ones). + elif letters and category not in CONTINUE_ON: + if letters: + aletters(u"".join(letters).strip() + BREAKER) + letters = [] + previous = LETTER + if digits: + adigits(u"".join(digits).strip()) + digits = [] + previous = NUMBER # Split if we find two pieces of punctuation in a row, even # if we should otherwise continue. - elif prev_category[0] in "P" and category[0] in "P": - broke = True - mode = UNKNOWN - - if broke and start is not None and last is not None: - # If we read two strings separated by weird punctuation, - # pretend the punctuation isn't there. - if this_mode == previous_mode == LETTER: - words[-1] += BREAKER + string[start:last+1] - else: - if this_mode == NUMBER and previous_mode == LETTER: - words[-1] += BREAKER - words.append(string[start:last+1]) - previous_mode = this_mode - - if broke: - start = i - last = None - if category[0] in "LN": - last = i - this_mode = mode - if start is not None and last is not None: - if this_mode == LETTER and previous_mode == LETTER and words: - words[-1] += BREAKER + string[start:last+1] + elif i and categories[i-1][0] in "P" and category[0] in "P": + if letters: + aletters(u"".join(letters)) + letters = [] + previous = LETTER + if digits: + adigits(u"".join(digits)) + digits = [] + previous = NUMBER + else: - if this_mode == NUMBER and previous_mode == LETTER and words: - words[-1] += BREAKER - words.append(string[start:last+1]) - return words + if digits: + digits.append(uchar) + elif letters: + letters.append(uchar) + + if letters and previous == LETTER and words: + word = stripends(words.pop()[1].strip()) + BREAKER + letters.insert(0, word) + previous = UNKNOWN + + if letters: + aletters(u"".join(letters)) + if digits: + adigits(u"".join(digits)) + + return [(i, key(w) if w else u'') for i, w in words] + +def numeric(orig, invalid=INFINITY): + """Parse a number out of a string. + + This function parses a unicode number out of the start of a + string. If a number cannot be found at the start, the 'invalid' + argument is returned. + + """ -def numeric(orig, invalid=float('inf')): if not orig: - return (invalid, '') + return invalid string = unicode(orig) - for c in string: - if c.isnumeric(): + for uchar in string: + if uchar.isnumeric(): break else: - return (invalid, orig) + return invalid mult = 1 while string[:1] == u"-" or string[:1] == u"+": @@ -117,37 +158,43 @@ def numeric(orig, invalid=float('inf')): string = string[1:] if not string[:1].isnumeric(): - return (invalid, orig) - - string = normalize_punc(string) + return invalid - # Early out if possible. - try: - return (float(string) * mult, orig) - except ValueError: - pass + string = normalize_number(string) - # Otherwise we need to do this the hard way. def _numeric(string): + """Interpreter a number as base 10.""" total = 0 - for c in string: - v = unicodedata.numeric(c) - if v >= 1 or v == 0: + for uchar in string: + number = unicodedata.numeric(uchar) + if number >= 1 or number == 0: total *= 10 - total += v + total += number return total try: whole, frac = string.split(".") whole = _numeric(whole) frac = _numeric(frac) / (10.0 ** len(frac)) - return (mult * (whole + frac), orig) + return mult * (whole + frac) except ValueError: - return (mult * _numeric(string), orig) + return mult * _numeric(string) + +def normalize_number(string): + """Normalize punctuation in a number. + + This function attempts to guess which characters in a number + represent grouping separators and which represent decimal + points. It returns a string that is valid to pass to Python's + float() routine (potentially, NaN, if nothing like a number is + found). + + """ + + string = unicode(string) + string = filter(lambda u: u.isnumeric() or u in KEEP_IN_NUMBERS, string) + string = string.strip(KEEP_IN_NUMBERS) -def normalize_punc(string): - string = unicode(string.strip(u",.'")) - string = filter(lambda u: u.isnumeric() or u in u",.'", string) commas = string.count(u",") stops = string.count(u".") quotes = string.count(u"'") @@ -164,7 +211,7 @@ def normalize_punc(string): quotes = 0 def normalize_two(a, b, string): - # One of each - assume the first is grouping, second is point. + """One of each - assume the first is grouping, second is point.""" a_idx = string.rindex(a) b_idx = string.rindex(b) if a_idx > b_idx: @@ -225,4 +272,4 @@ def normalize_punc(string): # Single stop, but no decimal - probably grouping. string = string.replace(u".", u"") - return string + return string or "NaN"