Exercise 3 solution

    import struct

    FIXED = struct.Struct("<4sBBH")      # magic, major, minor, flags: 8 bytes
    LENGTH = struct.Struct("<H")          # header_length field: 2 bytes
    MIN_HEADER = FIXED.size + LENGTH.size  # 10


    def write_file(extra=b"", payload=b""):
        header_length = MIN_HEADER + len(extra)
        return FIXED.pack(b"DEMO", 1, 0, 9) + LENGTH.pack(header_length) + extra + payload


    def read_file(raw):
        magic, major, minor, flags = FIXED.unpack_from(raw)
        if magic != b"DEMO":
            raise ValueError("bad magic")
        (header_length,) = LENGTH.unpack_from(raw, FIXED.size)
        if header_length < MIN_HEADER or header_length > len(raw):
            raise ValueError(f"impossible header_length {header_length}")
        return {"flags": flags, "header_length": header_length, "payload": raw[header_length:]}


    old = write_file(payload=b"DATA")
    new = write_file(extra=struct.pack("<I", 1700000000), payload=b"DATA")
    print(old.hex(" "))
    print(new.hex(" "))
    print(read_file(old))
    print(read_file(new))

    for label, raw in (("too small", old[:8] + LENGTH.pack(4) + b"DATA"),
                       ("too big", old[:8] + LENGTH.pack(500) + b"DATA")):
        try:
            read_file(raw)
        except ValueError as e:
            print(label + ":", e)

    # 44 45 4d 4f 01 00 09 00 0a 00 44 41 54 41
    # 44 45 4d 4f 01 00 09 00 0e 00 00 f1 53 65 44 41 54 41
    # {'flags': 9, 'header_length': 10, 'payload': b'DATA'}
    # {'flags': 9, 'header_length': 14, 'payload': b'DATA'}
    # too small: impossible header_length 4
    # too big: impossible header_length 500

(Run as written; the output matched the comments.)

Explanation: the header carries its own length, so a reader takes only the fields
it knows (the first 10 bytes) and uses header_length to find where the payload
starts. The newer file has 4 extra header bytes that this reader never
interprets, yet it still finds DATA correctly. The two error cases show why the
length is validated: a value below the 10 bytes this reader needs, or beyond the
end of the data, cannot be right.