# Words by the Numbers 6, Exercise 2
import pathlib, re, urllib.request

URL = "https://www.gutenberg.org/cache/epub/1404/pg1404.txt"
DISPUTED = set(range(49, 59)) | {62, 63}   # the twelve disputed essays
JOINT = {18, 19, 20}                       # joint work, left out of the comparisons

def load(path="federalist.txt"):
    p = pathlib.Path(path)
    if not p.exists():                     # download once, then reuse the saved copy
        req = urllib.request.Request(URL, headers={"User-Agent": "Mozilla/5.0"})
        p.write_bytes(urllib.request.urlopen(req).read())
    raw = p.read_text(encoding="utf-8-sig").replace("\r", "")
    parts = re.split(r"^FEDERALIST No\. (\d+)\s*$", raw, flags=re.M)
    papers = {}                            # essay number -> (author, list of words)
    for k in range(1, len(parts), 2):
        n, body = int(parts[k]), parts[k + 1]
        m = re.search(r"^(HAMILTON|MADISON|JAY)$", body, re.M)
        author = m.group(1) if m else "?"
        if n in DISPUTED: author = "DISPUTED"
        if n in JOINT: author = "JOINT"
        start = body.find("To the People")
        end = body.rfind("\nPUBLIUS")
        text = body[start:end if end > start else None]
        papers[n] = (author, re.findall(r"[a-z]+(?:'[a-z]+)?", text.lower()))
    return papers

import math
from collections import Counter

papers = load()
pool = {"HAMILTON": Counter(), "MADISON": Counter()}
for author, w in papers.values():
    if author in pool:
        pool[author].update(w)
total = {a: sum(c.values()) for a, c in pool.items()}

scored = []
for word in set(pool["HAMILTON"]) | set(pool["MADISON"]):
    h, m = pool["HAMILTON"][word], pool["MADISON"][word]
    if h + m >= 40:                                  # enough uses to mean something
        ratio = math.log2(((h + 0.5) / total["HAMILTON"]) / ((m + 0.5) / total["MADISON"]))
        scored.append((ratio, word, h, m))
scored.sort()
print("Madison-leaning (log2 of Hamilton rate / Madison rate; each step of 1 = twice as common)")
for ratio, word, h, m in scored[:8]:
    print(f"  {word:12} {ratio:6.2f}   Hamilton {h:4d}  Madison {m:4d}")
print("Hamilton-leaning")
for ratio, word, h, m in scored[::-1][:8]:
    print(f"  {word:12} {ratio:6.2f}   Hamilton {h:4d}  Madison {m:4d}")
print("words compared:", len(scored))

# Output when run (Python 3.14; Federalist text from the Project Gutenberg plain-text file):
# Madison-leaning (log2 of Hamilton rate / Madison rate; each step of 1 = twice as common)
#   department    -3.06   Hamilton   19  Madison   56
#   confederation  -2.76   Hamilton   23  Madison   55
#   departments   -2.36   Hamilton   20  Madison   36
#   congress      -2.11   Hamilton   39  Madison   59
#   articles      -1.99   Hamilton   28  Madison   39
#   powers        -1.91   Hamilton   98  Madison  128
#   forms         -1.87   Hamilton   18  Madison   23
#   according     -1.66   Hamilton   20  Madison   22
# Hamilton-leaning
#   kind           4.15   Hamilton   76  Madison    1
#   upon           4.11   Hamilton  372  Madison    7
#   community      3.27   Hamilton   69  Madison    2
#   courts         2.87   Hamilton  115  Madison    5
#   jury           2.81   Hamilton   70  Madison    3
#   matter         2.69   Hamilton   46  Madison    2
#   sometimes      2.46   Hamilton   39  Madison    2
#   thing          2.01   Hamilton   63  Madison    5
# words compared: 422