# Words by the Numbers 10, Exercise 3
# A detective that never crashes. Extend the detective so that it copes with a language that has
# no shelf (no keywords possible), then run it on three new passages: a poem, a Spanish novel, and
# a very short English passage. Say which parts of each verdict you would trust and which not.
import math, pathlib, random, re, statistics, urllib.request
from collections import Counter

LANG_TRAIN = {"English": 1342, "French": 17489, "German": 2229, "Spanish": 2000, "Italian": 1012}
AUTHORS = {"Austen": 1342, "Carroll": 11, "Wells": 35, "Doyle": 1661, "Dickens": 98}
OLD = {"A": (158, "Emma, Austen"), "B": (2852, "The Hound of the Baskervilles, Doyle"),
             "C": (2650, "Du cote de chez Swann, Proust"), "D": (22367, "Die Verwandlung, Kafka"),
             "E": (36, "The War of the Worlds, Wells"), "F": (1260, "Jane Eyre, Bronte (not a candidate)")}
MYSTERIES = {"G": (19, "The Song of Hiawatha, a poem by Longfellow"), "H": (1619, "La Celestina, in Spanish"),
             "I": (11, "30 words of Alice in Wonderland")}
SHELF = {"English": [1342, 11, 35, 1661, 98, 158, 12, 36, 2852, 730, 1260],
         "French": [17489, 2650], "German": [2229, 22367, 2407]}   # books used to measure how rare a word is

def load(number):
    path = pathlib.Path(f"pg{number}.txt")
    if not path.exists():                       # download once, then reuse the saved copy
        url = f"https://www.gutenberg.org/cache/epub/{number}/pg{number}.txt"
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        path.write_bytes(urllib.request.urlopen(req).read())
    raw = path.read_text(encoding="utf-8-sig").replace("\r", "")
    start = raw.index("\n", raw.index("*** START OF")) + 1
    return raw[start:raw.index("*** END OF")]

words = lambda text: re.findall(r"[^\W\d_]+(?:'[^\W\d_]+)?", text.lower())

# 1. language: cosine similarity of letter-pair profiles (as in the language snippet)
def pairs(text):
    t = " ".join(words(text))
    c = Counter(t[i:i + 2] for i in range(len(t) - 1) if " " not in t[i:i + 2])
    return {g: v / sum(c.values()) for g, v in c.items()}
def cosine(a, b):
    return sum(v * b.get(g, 0) for g, v in a.items()) / (math.sqrt(sum(v * v for v in a.values())) * math.sqrt(sum(v * v for v in b.values())))
lang_profiles = {lang: pairs(load(n)) for lang, n in LANG_TRAIN.items()}
def language(passage):
    p = pairs(passage)
    return max(lang_profiles, key=lambda lang: cosine(p, lang_profiles[lang]))

# 2. author: Delta over the 100 commonest words (as in the author snippet)
rates = lambda tokens, vocab: [1000 * Counter(tokens)[w] / len(tokens) for w in vocab]
train = {a: words(load(n)) for a, n in AUTHORS.items()}
vocab = [w for w, _ in Counter(sum(train.values(), [])).most_common(100)]
chunks = [rates(t[i:i + 2000], vocab) for t in train.values() for i in range(0, len(t) - 2000, 2000)]
mean = [statistics.fmean(col) for col in zip(*chunks)]
sd = [statistics.pstdev(col) or 1 for col in zip(*chunks)]
z = lambda r: [(x - m) / s for x, m, s in zip(r, mean, sd)]
author_z = {a: z(rates(t, vocab)) for a, t in train.items()}
def author(tokens):
    zp = z(rates(tokens, vocab))
    scores = {a: sum(abs(x - y) for x, y in zip(zp, za)) / len(zp) for a, za in author_z.items()}
    best = min(scores, key=scores.get)
    return best, scores[best]

# 3. topic: TF-IDF keywords, with rarity measured over 500-word pieces of the shelf for that language
pieces = {lang: [set(t[i:i + 500]) for t in map(lambda n: words(load(n)), books) for i in range(0, len(t) - 500, 500)]
          for lang, books in SHELF.items()}
def keywords(tokens, lang, k=5):
    tf, docs = Counter(tokens), pieces[lang]
    score = {w: tf[w] * math.log(len(docs) / (1 + sum(w in d for d in docs))) for w in tf if len(w) > 2}
    return sorted(score, key=score.get, reverse=True)[:k]

def report(tokens):
    lang = language(" ".join(tokens)[:300])
    line = f"language {lang}"
    if lang == "English":
        who, d = author(tokens)
        line += f"; author {who} (Delta {d:.2f}{', above the cut-off' if d > cut else ''})"
    if lang in pieces:
        line += f"; keywords {', '.join(keywords(tokens, lang))}"
    else:
        line += "; no keywords (no shelf for this language)"
    return line

random.seed(4)
cut = 1.21                                       # 95th percentile of best Delta for known authors
for label, (number, truth) in MYSTERIES.items():
    tokens = words(load(number))
    length = 30 if label == "I" else 1000
    s = random.randrange(len(tokens) - length)
    print(f"{label}: {report(tokens[s:s + length])}")
print()
for label, (number, truth) in MYSTERIES.items():
    print(f"  {label} was really: {truth}")

# Output when run (Python 3.14.0; texts from the Project Gutenberg plain-text files):
# G: language English; author Wells (Delta 1.21, above the cut-off); keywords hiawatha, mondamin, shuh, nokomis, wrestled
# H: language Spanish; no keywords (no shelf for this language)
# I: language English; author Wells (Delta 2.09, above the cut-off); keywords shrinking, fan, rapidly, avoid, holding
#
#   G was really: The Song of Hiawatha, a poem by Longfellow
#   H was really: La Celestina, in Spanish
#   I was really: 30 words of Alice in Wonderland