json1-3 Exercise 3: Encodings, BOMs, Line Endings and NaN ========================================================== Run with Python 3.14.0 on Windows 11: py ex3.py (plain py: do NOT add -X utf8 for the first part; this machine's default text encoding is cp1252) --- ex3.py --- # Exercise 3: files, encodings, line endings, NaN policy. Run with plain "py ex3.py" (NOT -X utf8) import json, pathlib, locale, sys, tempfile, os os.chdir(tempfile.mkdtemp()) print('preferred encoding:', locale.getpreferredencoding(False), '| UTF-8 mode:', sys.flags.utf8_mode) data = {'name': 'Zo' + chr(235), 'emoji': chr(0x1F600)} # 1. write and read with an explicit encoding with open('a.json', 'w', encoding='utf-8') as f: json.dump(data, f, ensure_ascii=False, indent=2) with open('a.json', encoding='utf-8') as f: print('1. explicit utf-8 round trip equal:', json.load(f) == data) # 2. read the same file WITHOUT an encoding with open('a.json') as f: print('2. open() with no encoding :', ascii(json.load(f)), '<- no error, wrong text') # 3. write without an encoding try: with open('b.json', 'w') as f: json.dump(data, f, ensure_ascii=False) except Exception as e: print('3. write, no encoding, ensure_ascii=False:', type(e).__name__) with open('c.json', 'w') as f: json.dump(data, f) print(' write, no encoding, default ensure_ascii:', pathlib.Path('c.json').read_text()) # 4. a BOM pathlib.Path('bom.json').write_bytes(b'\xef\xbb\xbf{"a":1}') try: with open('bom.json', encoding='utf-8') as f: json.load(f) except json.JSONDecodeError as e: print('4. BOM, utf-8 :', e.msg) with open('bom.json', encoding='utf-8-sig') as f: print(' BOM, utf-8-sig :', json.load(f)) # 5. line endings with open('crlf.json', 'w', encoding='utf-8') as f: json.dump({'a': [1]}, f, indent=2) with open('lf.json', 'w', encoding='utf-8', newline=chr(10)) as f: json.dump({'a': [1]}, f, indent=2) print('5. default newline:', ascii(pathlib.Path('crlf.json').read_bytes())) print(' newline=LF :', ascii(pathlib.Path('lf.json').read_bytes())) # 6. NaN policy print('6. dumps nan, default :', json.dumps([float('nan')])) try: json.dumps([float('nan')], allow_nan=False) except ValueError as e: print(' dumps allow_nan=False : ValueError:', e) def refuse(name): raise ValueError('refusing ' + name) try: json.loads('[NaN]', parse_constant=refuse) except ValueError as e: print(' loads parse_constant : ValueError:', e) # 7. command line pathlib.Path('t.json').write_text('{"b":1,"a":{"z":"' + chr(233) + '"}}', encoding='utf-8') import subprocess for args in [[], ['--sort-keys'], ['--indent', '1'], ['--tab'], ['--compact'], ['--no-ensure-ascii', '--compact']]: r = subprocess.run([sys.executable, '-X', 'utf8', '-m', 'json.tool', *args, 't.json'], capture_output=True, text=True, encoding='utf-8') print('7. json.tool ' + ' '.join(args).ljust(28) + ascii(r.stdout)) Output: preferred encoding: cp1252 | UTF-8 mode: 0 1. explicit utf-8 round trip equal: True 2. open() with no encoding : {'name': 'Zo\xc3\xab', 'emoji': '\xf0\u0178\u02dc\u20ac'} <- no error, wrong text 3. write, no encoding, ensure_ascii=False: UnicodeEncodeError write, no encoding, default ensure_ascii: {"name": "Zo\u00eb", "emoji": "\ud83d\ude00"} 4. BOM, utf-8 : Unexpected UTF-8 BOM (decode using utf-8-sig) BOM, utf-8-sig : {'a': 1} 5. default newline: b'{\r\n "a": [\r\n 1\r\n ]\r\n}' newline=LF : b'{\n "a": [\n 1\n ]\n}' 6. dumps nan, default : [NaN] dumps allow_nan=False : ValueError: Out of range float values are not JSON compliant: nan loads parse_constant : ValueError: refusing NaN 7. json.tool '{\n "b": 1,\n "a": {\n "z": "\\u00e9"\n }\n}\n' 7. json.tool --sort-keys '{\n "a": {\n "z": "\\u00e9"\n },\n "b": 1\n}\n' 7. json.tool --indent 1 '{\n "b": 1,\n "a": {\n "z": "\\u00e9"\n }\n}\n' 7. json.tool --tab '{\n\t"b": 1,\n\t"a": {\n\t\t"z": "\\u00e9"\n\t}\n}\n' 7. json.tool --compact '{"b":1,"a":{"z":"\\u00e9"}}\n' 7. json.tool --no-ensure-ascii --compact '{"b":1,"a":{"z":"\xe9"}}\n' WHY THIS WORKS AS AN ANSWER --------------------------- The script starts by printing the preferred encoding and the UTF-8 mode flag (cp1252, 0) so the reader knows which environment produced the results. Step 2 is the important one: reading a UTF-8 file with open() and no encoding raises no error and returns garbage ('Zoë' for "Zoe" with a diaeresis, and four odd characters for the emoji). Step 3 shows the same default failing loudly when writing with ensure_ascii=False, and passing silently when the output is ASCII-only (the default), which is why many programs never see the problem. The BOM needs utf-8-sig, and the CRLF/LF difference is shown in the raw bytes. The NaN lines show both ways to refuse it. The json.tool lines run each option and show the exact text. This was measured on Windows with Python 3.14.0 only: on Linux and macOS the default encoding is normally UTF-8, so step 2 may give correct text there, and newer Python versions may change the default. Not run: --no-indent and --json-lines (listed in --help only).