# Words by the Numbers 2, Exercise 2: compare an ideal 1/rank prediction with real counts.
import re
from collections import Counter
from pydoc_data.topics import topics

tokens = re.findall(r"[a-z']+", " ".join(topics.values()).lower())
counts = Counter(tokens)
n_tokens, n_types = len(tokens), len(counts)
norm = sum(1 / r for r in range(1, n_types + 1))
ranked = counts.most_common()
print("rank  word        actual  predicted  ratio")
for rank in (1, 2, 5, 10, 50, 100, 500, 1000):
    word, actual = ranked[rank - 1]
    predicted = n_tokens / (rank * norm)
    print(f"{rank:4d}  {word:<10} {actual:7d} {predicted:10.1f} {actual / predicted:6.2f}")

# Output when run (Python 3.14; the documentation-text counts depend on the Python version):
# rank  word        actual  predicted  ratio
#    1  the           5201     8601.4   0.60
#    2  a             2188     4300.7   0.51
#    5  in            1557     1720.3   0.91
#   10  are            760      860.1   0.88
#   50  see            220      172.0   1.28
#  100  added          121       86.0   1.41
#  500  similar         22       17.2   1.28
# 1000  uppercase        9        8.6   1.05