json1-12  Exercise 2: Clean with jq, Stream a Report in Node, Cross-Check with jq
=================================================================================

Run with Node 24.18.0 (Ajv 8.20.0, ajv-formats 3.0.1), Python 3.14.0 (jsonschema 4.26.0, jsonpatch 1.35, psutil 7.2.2), jq 1.8.2, curl 8.12.1 and sha256sum on Windows 11. All counts are from the single snapshot taken on 2026-10-12 (feed clock 19:05:41Z) and will differ for any later download.

--- 03_clean.jq ---
# Step 3: GeoJSON feature collection -> one flat record per earthquake.
# Rules: keep only type == "earthquake"; a null place becomes "(unknown)"; region is the text after the last ", " of place
# (or the whole place when there is no comma); times become ISO 8601 UTC strings; tsunami becomes a boolean; the point is split into lon, lat, depth_km.
def iso: (. / 1000 | floor | todate);
def region: if contains(", ") then (split(", ") | last) else . end;
.features[]
| select(.properties.type == "earthquake")
| .properties as $p
| ($p.place // "(unknown)") as $place
| {
    id: .id,
    time: ($p.time | iso),
    updated: ($p.updated | iso),
    mag: $p.mag,
    mag_type: $p.magType,
    place: $place,
    region: ($place | region),
    lon: .geometry.coordinates[0],
    lat: .geometry.coordinates[1],
    depth_km: .geometry.coordinates[2],
    status: $p.status,
    tsunami: ($p.tsunami == 1),
    sig: $p.sig,
    net: $p.net,
    url: $p.url
  }

--- 03_clean.sh ---
set -e
J=/c/lwdj/json/c4/bin/jq.exe
# -c: one record per line; -b: LF line endings on Windows (Chapter 9)
$J -b -c -f 03_clean.jq quakes.geojson > quakes.jsonl
echo "records written: $(wc -l < quakes.jsonl)"
echo "sha256 of quakes.jsonl: $(sha256sum quakes.jsonl | cut -d' ' -f1)"
echo "first record:"; head -1 quakes.jsonl
echo "CR bytes in the file (must be 0): $(tr -cd '\015' < quakes.jsonl | wc -c)"
echo "dropped by the type filter:"
$J -c '[.features[].properties.type | select(. != "earthquake")] | group_by(.) | map({(.[0]): length}) | add' quakes.geojson
echo "count check: kept + dropped = $(( $(wc -l < quakes.jsonl) + $($J '[.features[]|select(.properties.type != "earthquake")]|length' quakes.geojson) )) (features in the snapshot: $($J '.features|length' quakes.geojson))"
echo "the one feature with a null place is a $($J -r '.features[]|select(.id=="aka2026snusmn")|.properties.type' quakes.geojson), so the type filter dropped it;"
echo "records where the (unknown) rule fired: $($J -c 'select(.place=="(unknown)")' quakes.jsonl | wc -l)"
echo "region examples:"; $J -c '.region' quakes.jsonl | sort | uniq -c | sort -rn | head -5

Output:
records written: 9891
sha256 of quakes.jsonl: 35fb9a34a36a7df8d67804cb4bb6286c84d9a4e9c14bb15dc733a0ff36cbd81a
first record:
{"id":"aka2026ufzwkj","time":"2026-10-12T18:49:05Z","updated":"2026-10-12T18:50:09Z","mag":1.4,"mag_type":"ml","place":"10 km N of Susitna North, Alaska","region":"Alaska","lon":-149.852,"lat":62.256,"depth_km":14.2,"status":"automatic","tsunami":false,"sig":30,"net":"ak","url":"https://earthquake.usgs.gov/earthquakes/eventpage/aka2026ufzwkj"}
CR bytes in the file (must be 0): 0
dropped by the type filter:
{"explosion":72,"ice quake":22,"landslide":1,"mining explosion":2,"quarry blast":117}
count check: kept + dropped = 10105 (features in the snapshot: 10105)
the one feature with a null place is a quarry blast, so the type filter dropped it;
records where the (unknown) rule fired: 0
region examples:
   3323 "Alaska"
   2828 "CA"
    588 "Texas"
    490 "Hawaii"
    389 "Puerto Rico"

--- record.schema.json (the closed schema for one cleaned record) ---
{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://example.test/schemas/quake-record.json",
  "title": "One cleaned earthquake record (one line of quakes.jsonl)",
  "type": "object",
  "required": ["id", "time", "updated", "mag", "mag_type", "place", "region", "lon", "lat", "depth_km", "status", "tsunami", "sig", "net", "url"],
  "properties": {
    "id": { "type": "string", "pattern": "^[a-z0-9]+$" },
    "time": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$" },
    "updated": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$" },
    "mag": { "type": "number", "minimum": -2, "maximum": 10 },
    "mag_type": { "type": "string", "minLength": 1 },
    "place": { "type": "string", "minLength": 1 },
    "region": { "type": "string", "minLength": 1 },
    "lon": { "type": "number", "minimum": -180, "maximum": 180 },
    "lat": { "type": "number", "minimum": -90, "maximum": 90 },
    "depth_km": { "type": "number" },
    "status": { "enum": ["automatic", "reviewed"] },
    "tsunami": { "type": "boolean" },
    "sig": { "type": "integer", "minimum": 0, "maximum": 3000 },
    "net": { "type": "string", "minLength": 1 },
    "url": { "type": "string", "format": "uri" }
  },
  "additionalProperties": false
}

--- 04_report.mjs ---
// Step 4: stream quakes.jsonl one line at a time, validate every record, count, and write report.json and strong.jsonl (mag >= 5).
import fs from "node:fs";
import readline from "node:readline";
import Ajv2020 from "ajv/dist/2020.js";
import addFormats from "ajv-formats";

const input = process.argv[2] || "quakes.jsonl";
const ajv = new Ajv2020({ allErrors: false, strict: true }); addFormats(ajv);
const check = ajv.compile(JSON.parse(fs.readFileSync("record.schema.json", "utf8")));

const report = { source: input, lines: 0, blank_lines: 0, bad_json_lines: 0, schema_invalid_records: 0, valid_records: 0,
  by_magnitude: { "<2": 0, "2-3": 0, "3-4": 0, "4-5": 0, "5-6": 0, "6+": 0 }, by_status: {}, tsunami_flagged: 0,
  depth_km: { min: Infinity, max: -Infinity }, strongest: null };
const regions = new Map(); const dayCounts = new Map(); const seenIds = new Set(); let duplicateIds = 0;
const strong = fs.createWriteStream("strong.jsonl", { encoding: "utf8" });
const bucket = (m) => (m < 2 ? "<2" : m < 3 ? "2-3" : m < 4 ? "3-4" : m < 5 ? "4-5" : m < 6 ? "5-6" : "6+");

const rl = readline.createInterface({ input: fs.createReadStream(input, "utf8"), crlfDelay: Infinity });
for await (const line of rl) {
  report.lines++;
  if (!line.trim()) { report.blank_lines++; continue; }
  let r; try { r = JSON.parse(line); } catch { report.bad_json_lines++; continue; }
  if (!check(r)) { report.schema_invalid_records++; continue; }
  report.valid_records++;
  if (seenIds.has(r.id)) duplicateIds++; else seenIds.add(r.id);
  report.by_magnitude[bucket(r.mag)]++;
  report.by_status[r.status] = (report.by_status[r.status] || 0) + 1;
  if (r.tsunami) report.tsunami_flagged++;
  report.depth_km.min = Math.min(report.depth_km.min, r.depth_km); report.depth_km.max = Math.max(report.depth_km.max, r.depth_km);
  regions.set(r.region, (regions.get(r.region) || 0) + 1);
  const day = r.time.slice(0, 10); dayCounts.set(day, (dayCounts.get(day) || 0) + 1);
  if (!report.strongest || r.mag > report.strongest.mag) report.strongest = { id: r.id, mag: r.mag, place: r.place, time: r.time };
  if (r.mag >= 5) strong.write(line + "\n");
}
strong.end();
report.duplicate_ids = duplicateIds;
report.top_regions = [...regions].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)).slice(0, 10).map(([region, count]) => ({ region, count }));
const days = [...dayCounts].sort((a, b) => (a[0] < b[0] ? -1 : 1));
report.days_covered = days.length; report.busiest_day = days.reduce((a, b) => (b[1] > a[1] ? b : a));
report.checks = { valid_plus_invalid_equals_lines: report.valid_records + report.schema_invalid_records + report.bad_json_lines + report.blank_lines === report.lines,
  magnitude_buckets_sum_to_valid: Object.values(report.by_magnitude).reduce((a, b) => a + b, 0) === report.valid_records };
fs.writeFileSync("report.json", JSON.stringify(report, null, 2) + "\n");
console.log(JSON.stringify(report, null, 2));

Output of node 04_report.mjs (this is report.json):
{
  "source": "quakes.jsonl",
  "lines": 9891,
  "blank_lines": 0,
  "bad_json_lines": 0,
  "schema_invalid_records": 0,
  "valid_records": 9891,
  "by_magnitude": {
    "<2": 6898,
    "2-3": 1713,
    "3-4": 303,
    "4-5": 808,
    "5-6": 161,
    "6+": 8
  },
  "by_status": {
    "automatic": 1799,
    "reviewed": 8092
  },
  "tsunami_flagged": 5,
  "depth_km": {
    "min": -3.43,
    "max": 665.057
  },
  "strongest": {
    "id": "us6000u18k",
    "mag": 7.7,
    "place": "12 km WSW of Pitaloza Arriba, Panama",
    "time": "2026-10-09T17:56:06Z"
  },
  "duplicate_ids": 0,
  "top_regions": [
    {
      "region": "Alaska",
      "count": 3323
    },
    {
      "region": "CA",
      "count": 2828
    },
    {
      "region": "Texas",
      "count": 588
    },
    {
      "region": "Hawaii",
      "count": 490
    },
    {
      "region": "Puerto Rico",
      "count": 389
    },
    {
      "region": "New Mexico",
      "count": 296
    },
    {
      "region": "Washington",
      "count": 255
    },
    {
      "region": "Utah",
      "count": 213
    },
    {
      "region": "Nevada",
      "count": 122
    },
    {
      "region": "Oklahoma",
      "count": 111
    }
  ],
  "days_covered": 31,
  "busiest_day": [
    "2026-09-18",
    508
  ],
  "checks": {
    "valid_plus_invalid_equals_lines": true,
    "magnitude_buckets_sum_to_valid": true
  }
}

Resource use, measured with the Chapter 9 harness (m.py):
04_report.mjs (streaming)            0.43 s  peak      58 MB  rc=0  | }
(the harness prints the last line of the program's output after the exit status)

--- 05_crosscheck.sh (jq recomputes from the original GeoJSON; a control with one number changed must disagree) ---
# Step 5: recompute key numbers with a different tool (jq, straight from the ORIGINAL GeoJSON, not from quakes.jsonl)
# and compare them with report.json written by the Node script.
set -e
J=/c/lwdj/json/c4/bin/jq.exe
$J -c '
  [.features[] | select(.properties.type == "earthquake")] as $q
  | { valid_records: ($q | length),
      by_magnitude: { "<2": ([$q[] | select(.properties.mag < 2)] | length),
                      "2-3": ([$q[] | select(.properties.mag >= 2 and .properties.mag < 3)] | length),
                      "3-4": ([$q[] | select(.properties.mag >= 3 and .properties.mag < 4)] | length),
                      "4-5": ([$q[] | select(.properties.mag >= 4 and .properties.mag < 5)] | length),
                      "5-6": ([$q[] | select(.properties.mag >= 5 and .properties.mag < 6)] | length),
                      "6+":  ([$q[] | select(.properties.mag >= 6)] | length) },
      by_status: ($q | group_by(.properties.status) | map({(.[0].properties.status): length}) | add),
      tsunami_flagged: ([$q[] | select(.properties.tsunami == 1)] | length),
      strongest_id: ($q | max_by(.properties.mag) | .id) }' quakes.geojson > jq_numbers.json
cat jq_numbers.json
$J -e --slurpfile r report.json --slurpfile j jq_numbers.json -n '
  ($r[0]) as $r | ($j[0]) as $j
  | ($r.valid_records == $j.valid_records) and ($r.by_magnitude == $j.by_magnitude) and ($r.by_status == $j.by_status)
    and ($r.tsunami_flagged == $j.tsunami_flagged) and ($r.strongest.id == $j.strongest_id)' > /dev/null \
  && echo "jq and the Node report AGREE on counts, magnitude buckets, status, tsunami flags and the strongest event" \
  || echo "DISAGREEMENT between jq and the Node report"
echo "--- control: the same comparison with one number changed on purpose must disagree"
$J '.tsunami_flagged += 1' jq_numbers.json > jq_numbers_wrong.json
$J -e --slurpfile r report.json --slurpfile j jq_numbers_wrong.json -n '($r[0].tsunami_flagged == $j[0].tsunami_flagged)' > /dev/null && echo "control FAILED to disagree" || echo "control disagreed, as it should"

Output:
{"valid_records":9891,"by_magnitude":{"<2":6898,"2-3":1713,"3-4":303,"4-5":808,"5-6":161,"6+":8},"by_status":{"automatic":1799,"reviewed":8092},"tsunami_flagged":5,"strongest_id":"us6000u18k"}
jq and the Node report AGREE on counts, magnitude buckets, status, tsunami flags and the strongest event
--- control: the same comparison with one number changed on purpose must disagree
control disagreed, as it should

WHY THIS WORKS AS AN ANSWER
---------------------------
Nothing is lost silently: 9,891 kept + 214 dropped = 10,105 features, and the dropped ones are listed by type. The output has no carriage-return bytes (the check was tested first on a string known to contain one). The report arithmetic checks itself (valid + invalid + bad + blank equals lines; the magnitude buckets sum to the valid count), and the second tool, working from the original file rather than from the cleaned one, gets the same numbers; the deliberately wrong number is caught.
One honest finding: the rule that replaces a null place with '(unknown)' never fired, because the only null-place feature is a quarry blast that the type filter removes first. The region rule is crude (it gives 'CA' next to 'Alaska'). Not run: other cleaning choices, other tools for the second computation, larger windows of data.