X-Git-Url: https://git.korewanetadesu.com/?p=python-collate.git;a=blobdiff_plain;f=collate%2F_strings.py;h=aed2ba71c5394a0de89c5d05edb3ce65b3013efa;hp=fd18bb917008dcb56aa06246dc86d1a721fe601c;hb=90897d68d6f3b50fc0103b4bf84d2e13faa18ae9;hpb=53e1676b8d68cccd2b0692654d3871e44e0ba6b6

diff --git a/collate/_strings.py b/collate/_strings.py
index fd18bb9..aed2ba7 100644
--- a/collate/_strings.py
+++ b/collate/_strings.py
@@ -1,49 +1,107 @@
 import unicodedata
 
-def alnumsplit(string):
+CONTINUE_ON = frozenset([
+    "Ll", "Lm", "Lo", "Lt", "Lu",
+    "Mc", "Me", "Mn",
+    "Nd", "Nl", "No",
+    "Po",
+    "Zs",
+    ])
+
+UNKNOWN, LETTER, NUMBER = range(3)
+
+def sortemes(string):
+    """Generate a list of sortemes for the string.
+
+    A sorteme, by analogy with grapheme/morpheme/etc. is an atom of
+    sort information. This is larger than a word boundry but smaller
+    than a sentence boundry; roughly, a sorteme boundry occurs between
+    letters and numbers, between numbers and numbrs if 'too much'
+    punctuation exists in between, between lines.
+
+    There is no formal specification for sortemes; the goal of this
+    function is to provide good output for Collator.sortemekey.
+    """
+
+    words = []
+    if not string:
+        return words
     string = unicode(string)
-    strings = []
-    word = []
-    numeric = None
-    for char in string:
-        if numeric is None:
-            broke = False
-            if char.isnumeric():
-                numeric = True
-            elif char.isalpha():
-                numeric = False
-        elif numeric and char.isalpha():
+    start = None
+    last = None
+    mode = UNKNOWN
+    previous_mode = UNKNOWN
+    category = "XX"
+    for i, c in enumerate(string):
+        broke = False
+        prev_category = category
+        this_mode = mode
+        category = unicodedata.category(c)
+
+        # Split at the first letter following a number or
+        # non-continuing character.
+        if category[0] == "L":
+            if mode != LETTER:
+                broke = True
+                mode = LETTER
+
+        # Split at the first number following a non-number or
+        # non-continuing character.
+        elif category[0] == "N":
+            if mode != NUMBER:
+                broke = True
+                mode = NUMBER
+
+        # Split if we find a non-continuing character ("weird" ones).
+        elif category not in CONTINUE_ON:
             broke = True
-        elif not numeric and char.isnumeric():
+            mode = UNKNOWN
+
+        # Only certain punctuation allowed in numbers.
+        elif mode == NUMBER and category[0] == "P" and c not in "',._":
             broke = True
-        if broke:
-            if word:
-                strings.append(u"".join(word))
-                word = []
-            numeric = None
-        word.append(char)
-    if word:
-        strings.append(u"".join(word))
-    return strings
-
-def wordlike(string):
-    """Check if a string is 'word-like'.
-
-    Word-like strings contain at least one alphanumeric character.
-    """
+            mode = UNKNOWN
 
-    # Explicit loop is faster than:
-    #return any(map(type(string).isalnum, string))
+        # Split if we find two pieces of punctuation in a row, even
+        # if we should otherwise continue.
+        elif i > 0 and prev_category[0] == "P" and category[0] == "P":
+            broke = True
+            mode = UNKNOWN
+
+        if broke and start is not None and last is not None:
+            # If we read two strings separated by weird punctuation,
+            # pretend the punctuation isn't there.
+            if (this_mode == previous_mode == LETTER
+                and (category[0] == "P" or prev_category[0] == "P")
+                and words):
+                words[-1] += u" " + string[start:last+1]
+            else:
+                # This ensures "foo2 bar" sorts as ["foo ", 2, "bar"]
+                # Which sorts after ["foo", "bar"].
+                if this_mode == NUMBER and previous_mode == LETTER and words:
+                    words[-1] += u" "
+                words.append(string[start:last+1])
+                previous_mode = this_mode
 
-    for c in string:
-        if c.isalnum():
-            return True
-    else:
-        return False
+        if broke:
+            start = i
+            last = None
+        if category[0] in "LN":
+            last = i
+    this_mode = mode
+    if start is not None and last is not None:
+        if this_mode == LETTER and previous_mode == LETTER and words:
+            words[-1] += u" " + string[start:last+1]
+        else:
+            if this_mode == NUMBER and previous_mode == LETTER and words:
+                words[-1] += u" "
+            words.append(string[start:last+1])
+    return words
 
 def numeric(orig, invalid=float('inf')):
     if not orig:
         return (invalid, '')
+
     string = unicode(orig)
     for c in string:
         if c.isnumeric():
@@ -57,6 +115,11 @@ def numeric(orig, invalid=float('inf')):
             mult = -mult
         string = string[1:]
 
+    if not string[:1].isnumeric():
+        return (invalid, orig)
+
+    string = normalize_punc(string)
+
     # Early out if possible.
     try:
         return (float(string) * mult, orig)
@@ -64,13 +127,11 @@ def numeric(orig, invalid=float('inf')):
         pass
 
     # Otherwise we need to do this the hard way.
-    string = normalize_punc(string)
-
     def _numeric(string):
         total = 0
         for c in string:
             v = unicodedata.numeric(c)
-            if v >= 1:
+            if v >= 1 or v == 0:
                 total *= 10
             total += v
         return total
@@ -159,4 +220,8 @@ def normalize_punc(string):
         # Single quote, probably MM'SS", equivalent to a decimal point.
         string = string.replace(u"'", u".")
 
+    elif stops and string[-4:] == ".000":
+        # Single stop, but no decimal - probably grouping.
+        string = string.replace(u".", u"")
+
     return string