wordstat: Handle "invalid" UTF-8.

`pycld` is fussy where it comes to UTF-8 (see
mikemccand/chromium-compact-language-detector#22
and aboSamoor/polyglot#71).  This strips out
the characters that make `cld` choke.

Thanks to @andreoua for the suggested fix.

Author: sjlongland

hadsh/wordstat.py

@@ -5,14 +5,23 @@
 from string import punctuation
 
 
+def stripunprintable(s):
+    """
+    Strip non-printable characters
+    """
+    return ''.join(c for c in s if c.isprintable())
+
+
 def tokenise(html_text):
     """
     Return a list of words that appear in the text.
     """
     try:
         return list(
                 filter(lambda w : w not in punctuation,
-                    Text(html_to_text(html_text)).lower().words))
+                    Text(stripunprintable(
+                        html_to_text(html_text))
+                    ).lower().words))
     except ValueError:
         # Empty sequence?
         return []