json1-12  Exercise 1: Fetch One Snapshot, Validate It, Cross-Check the Validator
=================================================================================

Run with Node 24.18.0 (Ajv 8.20.0, ajv-formats 3.0.1), Python 3.14.0 (jsonschema 4.26.0, jsonpatch 1.35, psutil 7.2.2), jq 1.8.2, curl 8.12.1 and sha256sum on Windows 11. All counts are from the single snapshot taken on 2026-10-12 (feed clock 19:05:41Z) and will differ for any later download.

Data: USGS Earthquake Hazards Program, GeoJSON summary feed all_month. Licence: USGS says most of its information is in the public domain and asks for credit ("Source: U.S. Geological Survey"); I read this in policy pages found by search, not on the main copyright page (the fetch returned nothing and a direct request returned HTTP 403), so check the current terms before reuse.

--- 01_fetch.sh (the single download. I ran the identical curl command once; the rest of this exercise reads the saved files, so the script below was not run a second time) ---
# Step 1: fetch one snapshot of the USGS all_month feed and record what we got.
# One request only. The feed is refreshed every minute, so a second run gets different data.
set -e
J=/c/lwdj/json/c4/bin/jq.exe
URL="https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.geojson"
curl -sS -m 120 -A "working-with-json-course/1.0 (learning exercise)" -D hdr.txt -o quakes.geojson "$URL"
echo "bytes:  $(wc -c < quakes.geojson)"
echo "sha256: $(sha256sum quakes.geojson | cut -d' ' -f1)"
echo "status line: $(head -1 hdr.txt | tr -d '\r')"
echo "$(grep -i '^content-type' hdr.txt | tr -d '\r')"
echo "generated (feed clock): $($J -r ".metadata.generated/1000|todate" quakes.geojson)"
echo "metadata.count = $($J '.metadata.count' quakes.geojson); features in the file = $($J '.features|length' quakes.geojson)"
$J -e '.metadata.count == (.features|length)' quakes.geojson > /dev/null && echo "count check: OK" || echo "count check: MISMATCH"

--- 01_record.sh (the reporting half of 01_fetch.sh, run against the saved snapshot) ---
bytes:  7173808
sha256: 37f31144591d663c45ba9a8fb776a3530830258474a455b8441f264b8c44f05a
status line: HTTP/1.1 200 OK
Content-Type: application/json; charset=utf-8
generated (feed clock): 2026-10-12T19:05:41Z
metadata.count = 10105; features in the file = 10105
count check: OK

--- quake.schema.json ---
{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://example.test/schemas/quake-feed.json",
  "title": "USGS earthquake summary feed (the parts this pipeline relies on)",
  "type": "object",
  "required": ["type", "metadata", "features"],
  "properties": {
    "type": { "const": "FeatureCollection" },
    "metadata": {
      "type": "object",
      "required": ["generated", "count", "title"],
      "properties": {
        "generated": { "type": "integer", "minimum": 0 },
        "count": { "type": "integer", "minimum": 0 },
        "title": { "type": "string" }
      }
    },
    "features": { "type": "array", "items": { "$ref": "#/$defs/feature" } }
  },
  "$defs": {
    "feature": {
      "type": "object",
      "required": ["type", "id", "geometry", "properties"],
      "properties": {
        "type": { "const": "Feature" },
        "id": { "type": "string", "pattern": "^[a-z0-9]+$" },
        "geometry": {
          "type": "object",
          "required": ["type", "coordinates"],
          "properties": {
            "type": { "const": "Point" },
            "coordinates": {
              "type": "array",
              "prefixItems": [
                { "type": "number", "minimum": -180, "maximum": 180 },
                { "type": "number", "minimum": -90, "maximum": 90 },
                { "type": "number" }
              ],
              "items": false,
              "minItems": 3
            }
          }
        },
        "properties": {
          "type": "object",
          "required": ["mag", "place", "time", "updated", "status", "tsunami", "sig", "net", "type", "magType", "url"],
          "properties": {
            "mag": { "type": "number", "minimum": -2, "maximum": 10 },
            "place": { "type": "string", "minLength": 1 },
            "time": { "type": "integer", "minimum": 0 },
            "updated": { "type": "integer", "minimum": 0 },
            "status": { "enum": ["automatic", "reviewed"] },
            "tsunami": { "enum": [0, 1] },
            "sig": { "type": "integer", "minimum": 0, "maximum": 3000 },
            "net": { "type": "string", "minLength": 1 },
            "type": { "type": "string", "minLength": 1 },
            "magType": { "type": "string", "minLength": 1 },
            "url": { "type": "string", "format": "uri" }
          }
        }
      }
    }
  }
}

--- 02_validate.mjs (Ajv, per feature) ---
// Step 2: validate the downloaded document against quake.schema.json with Ajv, per feature, and report.
import fs from "node:fs";
import Ajv2020 from "ajv/dist/2020.js";
import addFormats from "ajv-formats";

const schema = JSON.parse(fs.readFileSync("quake.schema.json", "utf8"));
const doc = JSON.parse(fs.readFileSync(process.argv[2] || "quakes.geojson", "utf8"));

const ajv = new Ajv2020({ allErrors: true, strict: true });
addFormats(ajv);
const whole = ajv.compile(schema);
const featureOnly = ajv.compile({ $ref: schema.$id + "#/$defs/feature" });

// The whole-document verdict (stops reporting detail per feature), then a per-feature pass so each bad record is named.
const wholeOk = whole(doc);
console.log("whole document valid:", wholeOk, "| errors reported:", wholeOk ? 0 : whole.errors.length);

const bad = [];
for (const f of doc.features) if (!featureOnly(f)) bad.push({ id: f.id ?? "(no id)", errors: featureOnly.errors.map((e) => e.instancePath + " " + e.keyword + ": " + e.message) });
console.log("features:", doc.features.length, "| valid:", doc.features.length - bad.length, "| invalid:", bad.length);
const byKind = {};
for (const b of bad) for (const e of b.errors) byKind[e] = (byKind[e] || 0) + 1;
console.log("error kinds:", JSON.stringify(byKind));
for (const b of bad.slice(0, 5)) console.log("  e.g.", b.id, JSON.stringify(b.errors));

const countOk = doc.metadata.count === doc.features.length;
console.log("metadata.count equals number of features (a check JSON Schema cannot express):", countOk);
fs.writeFileSync("invalid_ids.json", JSON.stringify(bad.map((b) => b.id)) + "\n");

Output:
whole document valid: false | errors reported: 1
features: 10105 | valid: 10104 | invalid: 1
error kinds: {"/properties/place type: must be string":1}
  e.g. aka2026snusmn ["/properties/place type: must be string"]
metadata.count equals number of features (a check JSON Schema cannot express): true

--- 02_validate.py (a second validator: the invalid ids must match) ---
# Step 2 cross-check: the same schema, a different validator (jsonschema 4.26.0). The invalid ids must match Ajv's.
import json
from jsonschema import Draft202012Validator

schema = json.load(open("quake.schema.json", encoding="utf-8"))
doc = json.load(open("quakes.geojson", encoding="utf-8"))
Draft202012Validator.check_schema(schema)
feature_schema = {"$schema": schema["$schema"], "$ref": "#/$defs/feature", "$defs": schema["$defs"]}
v = Draft202012Validator(feature_schema)
bad = [f.get("id", "(no id)") for f in doc["features"] if not v.is_valid(f)]
ajv_bad = json.load(open("invalid_ids.json", encoding="utf-8"))
print("jsonschema invalid:", len(bad), "| Ajv invalid:", len(ajv_bad), "| same ids:", sorted(bad) == sorted(ajv_bad))
print("note: jsonschema ignores format unless a format checker is passed, so a bad url would not be caught here; none was caught by Ajv either")

Output:
jsonschema invalid: 1 | Ajv invalid: 1 | same ids: True
note: jsonschema ignores format unless a format checker is passed, so a bad url would not be caught here; none was caught by Ajv either

--- 02_control.mjs (eight copies of one real feature, six deliberately broken) ---
// Control for step 2: break copies of real features in known ways; the validator must reject exactly those.
import fs from "node:fs";
import Ajv2020 from "ajv/dist/2020.js";
import addFormats from "ajv-formats";
const schema = JSON.parse(fs.readFileSync("quake.schema.json", "utf8"));
const doc = JSON.parse(fs.readFileSync("quakes.geojson", "utf8"));
const ajv = new Ajv2020({ allErrors: true, strict: true }); addFormats(ajv); ajv.addSchema(schema);
const check = ajv.compile({ $ref: schema.$id + "#/$defs/feature" });
const base = () => structuredClone(doc.features[1]);
const cases = {
  "untouched copy": [(f) => f, true],
  "mag as a string": [(f) => { f.properties.mag = "1.4"; return f; }, false],
  "latitude 95": [(f) => { f.geometry.coordinates[1] = 95; return f; }, false],
  "only two coordinates": [(f) => { f.geometry.coordinates.pop(); return f; }, false],
  "unknown status": [(f) => { f.properties.status = "deleted"; return f; }, false],
  "time as a float": [(f) => { f.properties.time = 1.5; return f; }, false],
  "url not a URI": [(f) => { f.properties.url = "not a uri"; return f; }, false],
  "extra property (schema is open)": [(f) => { f.properties.extra = 1; return f; }, true],
};
let allAsExpected = true;
for (const [name, [mutate, expected]] of Object.entries(cases)) {
  const ok = check(mutate(base())); const as = ok === expected; allAsExpected &&= as;
  console.log(name.padEnd(34), "valid:", String(ok).padEnd(5), as ? "as expected" : "UNEXPECTED");
}
console.log("all control cases behaved as expected:", allAsExpected);

Output:
untouched copy                     valid: true  as expected
mag as a string                    valid: false as expected
latitude 95                        valid: false as expected
only two coordinates               valid: false as expected
unknown status                     valid: false as expected
time as a float                    valid: false as expected
url not a URI                      valid: false as expected
extra property (schema is open)    valid: true  as expected
all control cases behaved as expected: true

WHY THIS WORKS AS AN ANSWER
---------------------------
The snapshot is identified by size, SHA-256, the feed's own clock and a count check, so the results can be tied to exactly this data. The schema is open (new fields are allowed) but strict about the fields the pipeline needs. Ajv and jsonschema name the same single invalid feature (aka2026snusmn, whose place is null), so the verdict does not depend on one library. The control is what makes '10,104 valid' believable: six copies broken in six different ways were all rejected,
and the copy with an extra property, which the open schema permits, passed. Two things the schema cannot do are done in code (metadata.count equals the number of features). Not run: a second download, other feeds, other validators.