# Words by the Numbers 1, Exercise 2: two tokenisers compared
# Compare three ways of cutting the same sentence into words.
import re

text = "Don't pay $3.50 for a well-known e-mail tool; it's cheaper in Sao Paulo."
a = text.split()
b = re.findall(r"[a-z']+", text.lower())
c = re.findall(r"[a-z0-9]+(?:['.-][a-z0-9]+)*", text.lower())
for name, toks in (("split()", a), ("letters+apostrophe", b), ("keeps . and -", c)):
    print(f"{name:20} {len(toks):2d}  {toks}")

# Output when run:
# split()              13  ["Don't", 'pay', '$3.50', 'for', 'a', 'well-known', 'e-mail', 'tool;', "it's", 'cheaper', 'in', 'Sao', 'Paulo.']
# letters+apostrophe   14  ["don't", 'pay', 'for', 'a', 'well', 'known', 'e', 'mail', 'tool', "it's", 'cheaper', 'in', 'sao', 'paulo']
# keeps . and -        13  ["don't", 'pay', '3.50', 'for', 'a', 'well-known', 'e-mail', 'tool', "it's", 'cheaper', 'in', 'sao', 'paulo']