# Words by the Numbers 5, Exercise 3: next-word guessing with a held-out quarter of the topics
import re
from collections import Counter, defaultdict
from pydoc_data.topics import topics

def tokens(text):
    return re.findall(r"[a-z]+(?:'[a-z]+)?", text.lower())

names = sorted(topics)
train = [tokens(topics[n]) for i, n in enumerate(names) if i % 4 != 0]
test = [tokens(topics[n]) for i, n in enumerate(names) if i % 4 == 0]

c1 = defaultdict(Counter)
for d in train:
    for a, b in zip(d, d[1:]):
        c1[a][b] += 1

total = top1 = top3 = unseen = 0
for d in test:
    for a, b in zip(d, d[1:]):
        total += 1
        if a not in c1:
            unseen += 1
            continue
        guesses = [w for w, _ in c1[a].most_common(3)]
        top1 += guesses[0] == b
        top3 += b in guesses
print("test positions:", total, "| last word never seen in training:", f"{unseen / total:.0%}")
print(f"right with 1 guess: {top1 / total:.1%} | right within 3 guesses: {top3 / total:.1%}")

# Output when run (Python 3.14; the documentation-text figures depend on the Python version):
# test positions: 10336 | last word never seen in training: 4%
# right with 1 guess: 18.7% | right within 3 guesses: 29.4%