# Words by the Numbers 2, Exercise 3: where does the straight line hold? Slopes over rank bands.
import math
import re
from collections import Counter
from pydoc_data.topics import topics

def band_slope(freqs, lo, hi):
    xs = [math.log(r) for r in range(lo, hi + 1)]
    ys = [math.log(freqs[r - 1]) for r in range(lo, hi + 1)]
    mx, my = sum(xs) / len(xs), sum(ys) / len(ys)
    num = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
    return num / sum((x - mx) ** 2 for x in xs)

tokens = re.findall(r"[a-z']+", " ".join(topics.values()).lower())
freqs = sorted(Counter(tokens).values(), reverse=True)
for lo, hi in ((1, 10), (10, 100), (100, 1000), (1000, 2000)):
    print(f"ranks {lo:4d}-{hi:4d}: slope {band_slope(freqs, lo, hi):6.2f}")
print("types with count 1:", sum(1 for f in freqs if f == 1), "of", len(freqs))

# Output when run (Python 3.14; the documentation-text counts depend on the Python version):
# ranks    1-  10: slope  -0.76
# ranks   10- 100: slope  -0.84
# ranks  100-1000: slope  -1.21
# ranks 1000-2000: slope  -1.75
# types with count 1: 918 of 3445