# Words by the Numbers 10, Exercise 2
# Who gets confused with whom? Build the Delta profiles from the 100 commonest words, test 100
# random 1,000-word passages from a different book by each of the five authors, and print the
# confusion table (rows are the true author, columns the author named). Name the worst pair.
import pathlib, random, re, statistics, urllib.request
from collections import Counter

TRAIN = {"Austen": 1342, "Carroll": 11, "Wells": 35, "Doyle": 1661, "Dickens": 98}
TEST = {"Austen": 158, "Carroll": 12, "Wells": 36, "Doyle": 2852, "Dickens": 730}

def load(number):
    path = pathlib.Path(f"pg{number}.txt")
    if not path.exists():                       # download once, then reuse the saved copy
        url = f"https://www.gutenberg.org/cache/epub/{number}/pg{number}.txt"
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        path.write_bytes(urllib.request.urlopen(req).read())
    raw = path.read_text(encoding="utf-8-sig").replace("\r", "")
    start = raw.index("\n", raw.index("*** START OF")) + 1
    return raw[start:raw.index("*** END OF")]

def words(text):
    return re.findall(r"[a-z']+", text.lower())

def rates(tokens, vocab):                       # occurrences per 1,000 words for each word in vocab
    c = Counter(tokens)
    return [1000 * c[w] / len(tokens) for w in vocab]

random.seed(7)
train = {a: words(load(n)) for a, n in TRAIN.items()}
test = {a: words(load(n)) for a, n in TEST.items()}

def build(top_n):
    vocab = [w for w, _ in Counter(sum(train.values(), [])).most_common(top_n)]
    chunks = [rates(t[i:i + 2000], vocab) for t in train.values() for i in range(0, len(t) - 2000, 2000)]
    mean = [statistics.fmean(col) for col in zip(*chunks)]
    sd = [statistics.pstdev(col) or 1 for col in zip(*chunks)]
    z = lambda r: [(x - m) / s for x, m, s in zip(r, mean, sd)]
    return vocab, {a: z(rates(t, vocab)) for a, t in train.items()}, z

def delta(za, zb):
    return sum(abs(x - y) for x, y in zip(za, zb)) / len(za)

def attribute(passage, vocab, authors, z):
    zp = z(rates(passage, vocab))
    scores = {a: delta(zp, za) for a, za in authors.items()}
    best = min(scores, key=scores.get)
    return best, scores[best]


vocab, authors, z = build(100)
names = list(TRAIN)
table = {a: Counter() for a in names}
for a, t in test.items():
    for _ in range(100):
        s = random.randrange(len(t) - 1000)
        table[a][attribute(t[s:s + 1000], vocab, authors, z)[0]] += 1
print("rows: true author, columns: named as  " + "".join(f"{b:>9}" for b in names))
for a in names:
    print(f"  {a:18}" + "".join(f"{table[a][b]:>9}" for b in names))
wrong = [(table[a][b], a, b) for a in names for b in names if a != b]
print("most frequent mistake:", max(wrong))

# Output when run (Python 3.14.0; texts from the Project Gutenberg plain-text files):
# rows: true author, columns: named as     Austen  Carroll    Wells    Doyle  Dickens
#   Austen                   93        0        0        4        3
#   Carroll                   0      100        0        0        0
#   Wells                     0        3       90        1        6
#   Doyle                     0        0        7       76       17
#   Dickens                   4        6        0        4       86
# most frequent mistake: (17, 'Doyle', 'Dickens')