More tweaks; notably try to insert paragraph breaks rather than a separate Python...

[python-collate.git] / collate / icu / __init__.py

diff --git a/collate/icu/__init__.py b/collate/icu/__init__.py
index 6f9647e..5f3ec05 100644 (file)
--- a/collate/icu/__init__.py
+++ b/collate/icu/__init__.py
@@ -37,9 +37,11 @@ class Collator(collate._abcollator.Collator):
             self._breaker = _icu.WordBreaker("root")
 
     def words(self, string):
+        """Split the string along word boundries."""
         if isinstance(string, str):
-            string = string.decode(self.encoding, 'replace')
-        return filter(lambda u: not u.isspace(), self._breaker.words(string))
+            string = string.decode(self.encoding)
+        words = self._breaker.words(string)
+        return [w for w in words if not w.isspace()]
 
     def key(self, string):
         """Sort key for a string.