# Words by the Numbers 10, Exercise 1
# Language detective, shortest safe passage. Using letter-pair profiles trained on one book per
# language, test 100 random passages per language (from different books) at lengths 10 to 200
# characters. Print the accuracy for each language and the most common mistake at 20 characters.
import math, pathlib, random, re, urllib.request
from collections import Counter

TRAIN = {"English": 1342, "French": 17489, "German": 2229, "Spanish": 2000, "Italian": 1012}
TEST = {"English": [12, 36], "French": [2650], "German": [22367, 2407],
        "Spanish": [49836], "Italian": [45334]}      # different books from the training ones

def load(number):
    path = pathlib.Path(f"pg{number}.txt")
    if not path.exists():                       # download once, then reuse the saved copy
        url = f"https://www.gutenberg.org/cache/epub/{number}/pg{number}.txt"
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        path.write_bytes(urllib.request.urlopen(req).read())
    raw = path.read_text(encoding="utf-8-sig").replace("\r", "")
    start = raw.index("\n", raw.index("*** START OF")) + 1
    return raw[start:raw.index("*** END OF")]

def clean(text):
    return re.sub(r"[\W\d_]+", " ", text.lower()).strip()

def pairs(text):
    c = Counter(text[i:i + 2] for i in range(len(text) - 1) if " " not in text[i:i + 2])
    return {g: v / sum(c.values()) for g, v in c.items()}

def cosine(a, b):
    if not a:                                   # a very short passage can have no letter pair at all
        return 0.0
    dot = sum(v * b.get(g, 0) for g, v in a.items())
    return dot / (math.sqrt(sum(v * v for v in a.values())) * math.sqrt(sum(v * v for v in b.values())))

random.seed(5)
profiles = {lang: pairs(clean(load(n))) for lang, n in TRAIN.items()}
pools = {lang: [clean(load(n)) for n in nums] for lang, nums in TEST.items()}
mistakes = Counter()
print("accuracy by passage length (characters):")
print("  " + " " * 8 + "".join(f"{length:>7}" for length in (10, 20, 50, 100, 200)))
table = {lang: [] for lang in pools}
for length in (10, 20, 50, 100, 200):
    for lang, books in pools.items():
        right = 0
        for _ in range(100):
            book = random.choice(books)
            s = random.randrange(len(book) - length)
            p = pairs(book[s:s + length])
            guess = max(profiles, key=lambda other: cosine(p, profiles[other]))
            right += guess == lang
            if length == 20 and guess != lang:
                mistakes[(lang, guess)] += 1
        table[lang].append(right)
for lang, row in table.items():
    print(f"  {lang:8}" + "".join(f"{v:>6}%" for v in row))
print("most common mistakes at 20 characters (true -> guessed):", mistakes.most_common(3))

# Output when run (Python 3.14.0; texts from the Project Gutenberg plain-text files):
# accuracy by passage length (characters):
#                10     20     50    100    200
#   English     66%    77%    92%   100%   100%
#   French      62%    74%    90%   100%   100%
#   German      75%    88%    99%   100%   100%
#   Spanish     56%    62%    87%    93%   100%
#   Italian     56%    70%    88%    96%   100%
# most common mistakes at 20 characters (true -> guessed): [(('Spanish', 'Italian'), 25), (('Italian', 'Spanish'), 16), (('French', 'Spanish'), 11)]