From dd07335d3922d4c63ec55b2fcc0585a44b63714c Mon Sep 17 00:00:00 2001 From: Golumpa Date: Wed, 15 Jul 2026 17:30:19 +0100 Subject: [PATCH 1/8] Improve test suite Improves the existing test suite by splitting the single test file into multiple files. Also adds the ability for other people to use the tests with some example data. --- tests/__init__.py | 1 + tests/fixtures/README.md | 23 + tests/fixtures/synthetic_history_network.json | 322 +++++ tests/support.py | 93 ++ tests/test_arc_decoding.py | 120 ++ tests/test_boundary_export.py | 1046 ----------------- tests/test_capture_backends.py | 92 ++ tests/test_export_contract.py | 108 ++ tests/test_live_session.py | 214 ++++ tests/test_mappings_and_updates.py | 122 ++ tests/test_network_fixture.py | 74 ++ tests/test_protocol_decoding.py | 125 ++ tests/test_uid_compatibility.py | 153 +++ 13 files changed, 1447 insertions(+), 1046 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/README.md create mode 100644 tests/fixtures/synthetic_history_network.json create mode 100644 tests/support.py create mode 100644 tests/test_arc_decoding.py delete mode 100644 tests/test_boundary_export.py create mode 100644 tests/test_capture_backends.py create mode 100644 tests/test_export_contract.py create mode 100644 tests/test_live_session.py create mode 100644 tests/test_mappings_and_updates.py create mode 100644 tests/test_network_fixture.py create mode 100644 tests/test_protocol_decoding.py create mode 100644 tests/test_uid_compatibility.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..3f4d23c --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for the NTE history exporter.""" diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..7c54fd3 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,23 @@ +# Synthetic network fixture + +`synthetic_history_network.json` is the committed end-to-end packet fixture for +the exporter test suite. It contains UDP payloads for Permanent Board and Arc +history requests/responses, plus isolated protocol edge cases. + +The fixture is safe to publish: + +- endpoints use the RFC 5737 documentation ranges `192.0.2.0/24` and + `198.51.100.0/24`; +- packet and history timestamps are deterministic synthetic values; +- it contains no game user UID, account/session data, cookies, tokens, or raw + capture metadata; +- the test suite checks every payload with the UID extractor before accepting + it. + +Tests load the fixture through helpers in `tests/support.py`. The `replay` +scenario is fed through `LiveHistorySession`; `protocol_sample` packets cover +points gifts, chase rewards, batched pages, and bit-packed responses. + +Do not replace this file with a real `.pcap`, `.flows`, or exported account +history. Add new cases by constructing the smallest relevant payload, replacing +all timestamps and endpoints, and extending the privacy assertions. diff --git a/tests/fixtures/synthetic_history_network.json b/tests/fixtures/synthetic_history_network.json new file mode 100644 index 0000000..4f3e531 --- /dev/null +++ b/tests/fixtures/synthetic_history_network.json @@ -0,0 +1,322 @@ +{ + "type": "nte-synthetic-network-fixture", + "schema_version": 1, + "description": "Synthetic UDP history transcript for tests. Contains no real account or network identity data.", + "privacy": { + "synthetic": true, + "contains_user_uid": false, + "contains_raw_account_session": false, + "addresses": "RFC 5737 documentation ranges", + "timestamps": "deterministic synthetic values" + }, + "local_ip": "192.0.2.10", + "expected": { + "permanent_records": 25, + "arc_records": 25, + "permanent_first_uid": "bdf0395ad3fb21f89b62ca92aaef7e36", + "arc_first_uid": "221b6accc3de16a145293b8804761c7c" + }, + "packets": [ + { + "type": "packet", + "label": "permanent-page-1-request", + "scenario": "replay", + "timestamp": 1.0, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "00000000000000000000000000000000000000000000000000000000000000040000007c100000000400000000" + }, + { + "type": "packet", + "label": "permanent-page-1-response", + "scenario": "replay", + "timestamp": 1.01, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23" + }, + { + "type": "packet", + "label": "permanent-page-2-request", + "scenario": "replay", + "timestamp": 1.02, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "00000000000000000000000000000000000000000000000000000000000000080000007c100000000400000000" + }, + { + "type": "packet", + "label": "permanent-page-2-response", + "scenario": "replay", + "timestamp": 1.03, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23" + }, + { + "type": "packet", + "label": "permanent-page-3-request", + "scenario": "replay", + "timestamp": 1.04, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000c0000007c100000000400000000" + }, + { + "type": "packet", + "label": "permanent-page-3-response", + "scenario": "replay", + "timestamp": 1.05, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23" + }, + { + "type": "packet", + "label": "permanent-page-4-request", + "scenario": "replay", + "timestamp": 1.06, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "00000000000000000000000000000000000000000000000000000000000000100000007c100000000400000000" + }, + { + "type": "packet", + "label": "permanent-page-4-response", + "scenario": "replay", + "timestamp": 1.07, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23" + }, + { + "type": "packet", + "label": "permanent-page-5-request", + "scenario": "replay", + "timestamp": 1.08, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "00000000000000000000000000000000000000000000000000000000000000140000007c100000000400000000" + }, + { + "type": "packet", + "label": "permanent-page-5-response", + "scenario": "replay", + "timestamp": 1.09, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23" + }, + { + "type": "packet", + "label": "arc-page-1-request", + "scenario": "replay", + "timestamp": 1.1, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "0000000000000000000000000000000000000000000000000c080000000200000000" + }, + { + "type": "packet", + "label": "arc-page-1-response", + "scenario": "replay", + "timestamp": 1.11, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511" + }, + { + "type": "packet", + "label": "arc-page-2-request", + "scenario": "replay", + "timestamp": 1.12, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "0000000000000000000000000000000000000000000000000c080000000400000000" + }, + { + "type": "packet", + "label": "arc-page-2-response", + "scenario": "replay", + "timestamp": 1.1300000000000001, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511" + }, + { + "type": "packet", + "label": "arc-page-3-request", + "scenario": "replay", + "timestamp": 1.1400000000000001, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "0000000000000000000000000000000000000000000000000c080000000600000000" + }, + { + "type": "packet", + "label": "arc-page-3-response", + "scenario": "replay", + "timestamp": 1.1500000000000001, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511" + }, + { + "type": "packet", + "label": "arc-page-4-request", + "scenario": "replay", + "timestamp": 1.1600000000000001, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "0000000000000000000000000000000000000000000000000c080000000800000000" + }, + { + "type": "packet", + "label": "arc-page-4-response", + "scenario": "replay", + "timestamp": 1.1700000000000002, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511" + }, + { + "type": "packet", + "label": "arc-page-5-request", + "scenario": "replay", + "timestamp": 1.1800000000000002, + "protocol": "udp", + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 49152, + "dst_port": 30031, + "payload_hex": "0000000000000000000000000000000000000000000000000c080000000a00000000" + }, + { + "type": "packet", + "label": "arc-page-5-response", + "scenario": "replay", + "timestamp": 1.1900000000000002, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511" + }, + { + "type": "packet", + "label": "limited-points-gift-1", + "scenario": "protocol_sample", + "timestamp": 11.0, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e4050000140000000000000014000000c4c0c8c000000000000400000014000000c4c0c8c0004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23" + }, + { + "type": "packet", + "label": "limited-chase-reward", + "scenario": "protocol_sample", + "timestamp": 12.0, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009806000014000000fcffffff4b00000010a58d957dd1a58dad95d17dc1c4b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c4004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23" + }, + { + "type": "packet", + "label": "limited-batched-pages", + "scenario": "protocol_sample", + "timestamp": 13.0, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e805000014000000140000002c00000098bdc9ad7db9bdb9bdcd0100000000040000002c00000098bdc9ad7db9bdb9bdcd014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230400000014000000c4c0c8c000000000000400000014000000c4c0c8c0004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100000002c00000098bdc9ad7db9bdb9bdcd0100000000040000002c00000098bdc9ad7db9bdb9bdcd014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000002800000098bdc9ad7dd9a5b9950100000000080000003c00000010a58d957dd1a58dad95d17dc1c4002800000098bdc9ad7dd9a5b995014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000003400000098bdc9ad7d91d5cdd189a5b90100000000040000003400000098bdc9ad7d91d5cdd189a5b9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100126675671c6e0ecfcb769070000000c08000000100000003e0800000004000000486c0000001835bdb9bdc1bdb1e531bdd1d195c9e549958dbdc9911185d18501000406000014000000100000003000000098bdc9ad7de585bd9185bd0100000000040000003000000098bdc9ad7de585bd9185bd014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230c00000014000000c4c0d4c80000000000080000003c00000010a58d957dd1a58dad95d17dc1c40014000000c4c0d4c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23140000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b231800000014000000c4c0c0e000000000000400000014000000c4c0c0e0004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100000003c00000098bdc9ad7d41c9bdad85c9e5bdb90100000000040000003c00000098bdc9ad7d41c9bdad85c9e5bdb9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23" + }, + { + "type": "packet", + "label": "limited-bitpacked-source", + "scenario": "protocol_sample", + "timestamp": 14.0, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004c06000014000000180000003c00000098bdc9ad7d41c9bdad85c9e5bdb90100000000080000003c00000010a58d957dd1a58dad95d17dc1c4003c00000098bdc9ad7d41c9bdad85c9e5bdb9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230000000014000000c4c0c4e400000000000400000014000000c4c0c4e4004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000003400000098bdc9ad7d91d5cdd189a5b90100000000080000003c00000010a58d957dd1a58dad95d17dc1c4003400000098bdc9ad7d91d5cdd189a5b9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100000003c00000098bdc9ad7d85c1c1b1a585b98d950100000000040000003c00000098bdc9ad7d85c1c1b1a585b98d95014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b231000000014000000c4c0cccc00000000000400000014000000c4c0cccc004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23005406000014000000080000004000000098bdc9ad7dddd5a1d585add585b99d0100000000040000004000000098bdc9ad7dddd5a1d585add585b99d014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230c0000002c00000010a58d95b1a5b5a5d1950100000000040000002c00000010a58d95b1a5b5a5d195014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000003400000098bdc9ad7d91d5cdd189a5b90100000000040000003400000098bdc9ad7d91d5cdd189a5b9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23180000004800000010a58d957dd1a58dad95d17dc1c8b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23180000004800000010a58d957dd1a58dad95d17dc1c8b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23" + }, + { + "type": "packet", + "label": "limited-bitpacked-response", + "scenario": "protocol_sample", + "timestamp": 15.0, + "protocol": "udp", + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 30031, + "dst_port": 49152, + "payload_hex": "15000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080c9000080020000000300008007000000b337b9b52f28b9b7b530b9bc373700000000000100008007000000a2b4b1b22fbab4b1b532ba2f98188007000000b337b9b52f28b9b7b530b9bc37378009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b62710400000080020000801898981c000000008000000080020000801898981c8009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271840000008006000000b337b9b52fb2ba393ab1343700000000000100008007000000a2b4b1b22fbab4b1b532ba2f98188006000000b337b9b52fb2ba393ab134378009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271040200008007000000b337b9b5af303838b6b430b7b13200000000800000008007000000b337b9b5af303838b6b430b7b1328009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b62710402000080020000801898991900000000800000008002000080189899198009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b62710480ca000080020000000100000008000000b337b9b5afbb3ab4bab0b5ba30b73300000000800000000008000000b337b9b5afbb3ab4bab0b5ba30b7338009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271840100008005000000a2b4b132b6b4b634ba3200000000800000008005000000a2b4b132b6b4b634ba328009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271840000008006000000b337b9b52fb2ba393ab1343700000000800000008006000000b337b9b52fb2ba393ab134378009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271040300000009000000a2b4b1b22fbab4b1b532ba2f181996191800000000800000008007000000a2b4b1b22fbab4b1b532ba2f18198009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271040300000009000000a2b4b1b22fbab4b1b532ba2f181996191800000000800000008007000000a2b4b1b22fbab4b1b532ba2f18198009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b627104" + } + ] +} + diff --git a/tests/support.py b/tests/support.py new file mode 100644 index 0000000..d64467d --- /dev/null +++ b/tests/support.py @@ -0,0 +1,93 @@ +import csv +import json +import sys +import unittest +from pathlib import Path +from unittest.mock import Mock, patch +from tempfile import TemporaryDirectory + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +FIXTURES = ROOT / "tests" / "fixtures" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from nte_history_exporter import __version__ +from nte_history_exporter.decoder.boundary import annotate_groups, make_uid +from nte_history_exporter.constants import LIMITED_CHARACTER_MARKER, MARKER +from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 +from nte_history_exporter.decoder.protocol import decode_response_records, history_request_kind +from nte_history_exporter.constants import POOL_META +from nte_history_exporter.mappings import ARC_META, CHARACTERS, ITEMS, REWARDS_BY_ID +from nte_history_exporter.decoder.protocol import decode_reward_key, infer_reward_type +from nte_history_exporter.decoder.user_uid import extract_user_uid +from nte_history_exporter.export.csv_export import write_csv +from nte_history_exporter.decoder.arc import ( + arc_request_page, + build_arc_rows_from_pairs, + decode_arc_key, + decode_arc_timestamp, + is_arc_history_request, + make_arc_uid, + parse_arc_response, +) +from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket +from nte_history_exporter.live_capture.libpcap import ( + DLT_EN10MB, + DLT_LINUX_SLL, + DLT_LINUX_SLL2, + DLT_LOOP, + DLT_RAW, + _extract_ipv4_frame, + _load_library, + LibpcapUnavailable, +) +from nte_history_exporter.live_capture.windows_raw import parse_ipv4_packet +from nte_history_exporter.live_capture.backends import open_capture_backend +from nte_history_exporter.export.json_export import build_export_json +from nte_history_exporter.live_capture.runner import export_paths +from nte_history_exporter.pool_mappings import load_pool_mappings, pool_meta_from_mapping +from nte_history_exporter.update_check import UpdateInfo, check_for_update, is_newer_version + + +def decode_single_record(record_hex): + return decode_response_records(bytes(0x50) + bytes.fromhex(record_hex))[0] + + +def load_network_fixture(): + path = FIXTURES / "synthetic_history_network.json" + with path.open(encoding="utf-8") as f: + return json.load(f) + + +def fixture_packets(scenario=None): + fixture = load_network_fixture() + packets = fixture["packets"] + if scenario is not None: + packets = [packet for packet in packets if packet["scenario"] == scenario] + return [ + UdpPacket( + timestamp=packet["timestamp"], + src_ip=packet["src_ip"], + dst_ip=packet["dst_ip"], + src_port=packet["src_port"], + dst_port=packet["dst_port"], + payload=bytes.fromhex(packet["payload_hex"]), + protocol=packet["protocol"], + ) + for packet in packets + ] + + +def fixture_payload(label): + fixture = load_network_fixture() + packet = next(packet for packet in fixture["packets"] if packet["label"] == label) + return bytes.fromhex(packet["payload_hex"]) + + +def fixture_session(): + fixture = load_network_fixture() + session = LiveHistorySession(fixture["local_ip"]) + for packet in fixture_packets("replay"): + session.process_packet(packet) + return session diff --git a/tests/test_arc_decoding.py b/tests/test_arc_decoding.py new file mode 100644 index 0000000..4a422d9 --- /dev/null +++ b/tests/test_arc_decoding.py @@ -0,0 +1,120 @@ +from tests.support import * # noqa: F401,F403 + + +class ArcDecodingTests(unittest.TestCase): + def test_arc_key_timestamp_and_uid_match_fixture(self): + fixture = load_network_fixture() + row = fixture_session().build_rows("arc_miracle_box")[0] + self.assertEqual(decode_arc_key(bytes.fromhex(row["reward_key_hex"])), "fork_nonos") + _ticks, _unix, decoded = decode_arc_timestamp(bytes.fromhex(row["timestamp_raw_hex"])) + self.assertEqual(decoded, "2030-01-02 00:00:00") + self.assertEqual(row["uid"], fixture["expected"]["arc_first_uid"]) + + def test_arc_response_parser_matches_fixture_first_page(self): + decoded = parse_arc_response(fixture_payload("arc-page-1-response")) + self.assertEqual(len(decoded), 5) + self.assertEqual([row["reward_id"] for row in decoded], ["fork_nonos"] * 5) + self.assertEqual(decoded[0]["reward_type"], "arc") + self.assertEqual(decode_arc_key(bytes.fromhex(decoded[0]["reward_key_hex"])), "fork_nonos") + + def test_arc_response_parser_rejects_invalid_timestamp_noise(self): + response = bytearray(0x4C) + response += (10).to_bytes(4, "little") + response += bytes.fromhex("ccdee4d6be") + response += (8).to_bytes(4, "little") + response += b"garb" + response += (0xFFFFFFFFFFFFFFFF).to_bytes(8, "little") + + self.assertEqual(parse_arc_response(bytes(response)), []) + + def test_arc_partial_timestamp_group_is_exported_without_warning(self): + decoded = fixture_session().build_rows("arc_miracle_box") + exported = [row for row in decoded if row["export_record"] is True] + + # The oldest group is a 10-pull split by stopping at page 5 (5 of 10 rows). + # Its captured prefix is ordinal-stable, so every row is exported with a + # stable UID and no warning. + self.assertEqual(len(decoded), 25) + self.assertEqual(len(exported), 25) + self.assertTrue(all(row["uid"] for row in decoded)) + self.assertTrue(all(row["uid_status"] == "stable" for row in decoded)) + + def test_arc_incomplete_prefix_keeps_stable_uids(self): + pairs = fixture_session().pairs_for_kind("arc_miracle_box") + full_pairs = pairs[:2] + trunc_pairs = pairs[:1] + ts = parse_arc_response(fixture_payload("arc-page-1-response"))[0]["timestamp_raw_hex"] + full = [r for r in build_arc_rows_from_pairs(full_pairs) if r["timestamp_raw_hex"] == ts] + trunc = [r for r in build_arc_rows_from_pairs(trunc_pairs) if r["timestamp_raw_hex"] == ts] + self.assertTrue(trunc) + self.assertEqual([r["uid"] for r in trunc], [r["uid"] for r in full[: len(trunc)]]) + + def test_arc_row_builder_accepts_live_pairs_with_kind(self): + response = fixture_payload("arc-page-1-response") + decoded = build_arc_rows_from_pairs([(1, 2, 1, 1.0, 2, 1.1, response, "arc_miracle_box")]) + self.assertEqual(len(decoded), 5) + self.assertEqual(decoded[0]["reward_id"], "fork_nonos") + + def test_arc_export_is_shared_pity(self): + rows = fixture_session().build_rows("arc_miracle_box") + export = build_export_json(rows, []) + self.assertEqual(export["banner"]["id"], "Arc_MiracleBox") + self.assertIs(export["banner"]["shared_pity"], True) + + def test_group_detection_counts_only_dice_records_but_uid_ordinals_keep_all_rows(self): + rows = [ + { + "page": 1, + "timestamp_raw_hex": "aa", + "timestamp_decoded": "2026-01-01 00:00:00", + "result_type": "dice", + "dice": 1, + "reward_key_hex": "k1", + "quantity": 1, + }, + { + "page": 1, + "timestamp_raw_hex": "aa", + "timestamp_decoded": "2026-01-01 00:00:00", + "result_type": "points_gift", + "dice": 0, + "reward_key_hex": "k2", + "quantity": 1, + }, + { + "page": 1, + "timestamp_raw_hex": "aa", + "timestamp_decoded": "2026-01-01 00:00:00", + "result_type": "chase_reward", + "dice": -4, + "reward_key_hex": "k3", + "quantity": 30, + }, + { + "page": 1, + "timestamp_raw_hex": "aa", + "timestamp_decoded": "2026-01-01 00:00:00", + "result_type": "dice", + "dice": 2, + "reward_key_hex": "k4", + "quantity": 1, + }, + { + "page": 1, + "timestamp_raw_hex": "bb", + "timestamp_decoded": "2026-01-01 00:01:00", + "result_type": "dice", + "dice": 3, + "reward_key_hex": "k5", + "quantity": 1, + }, + ] + + annotated = annotate_groups(rows) + + # Ordinals cover every row in the group, but the dice-only count drives + # timestamp_group_size_seen (2 dice in the 4-record group). + self.assertEqual([row["timestamp_group_ordinal"] for row in annotated[:4]], [0, 1, 2, 3]) + self.assertEqual({row["timestamp_group_size_seen"] for row in annotated[:4]}, {2}) + self.assertEqual({row["timestamp_group_record_size_seen"] for row in annotated[:4]}, {4}) + self.assertTrue(all(row["export_record"] for row in annotated)) diff --git a/tests/test_boundary_export.py b/tests/test_boundary_export.py deleted file mode 100644 index 7d82cc9..0000000 --- a/tests/test_boundary_export.py +++ /dev/null @@ -1,1046 +0,0 @@ -import csv -import json -import sys -import unittest -from pathlib import Path -from unittest.mock import Mock, patch -from tempfile import TemporaryDirectory - -ROOT = Path(__file__).resolve().parents[1] -SRC = ROOT / "src" -EXPORTS = ROOT / "exports" -V7_EXPORTS = EXPORTS / "1" -ARC_EXPORTS = EXPORTS / "arc" -if str(SRC) not in sys.path: - sys.path.insert(0, str(SRC)) - -from nte_history_exporter import __version__ -from nte_history_exporter.decoder.boundary import annotate_groups, make_uid -from nte_history_exporter.constants import LIMITED_CHARACTER_MARKER, MARKER -from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 -from nte_history_exporter.decoder.protocol import decode_response_records, history_request_kind -from nte_history_exporter.constants import POOL_META -from nte_history_exporter.mappings import ARC_META, CHARACTERS, ITEMS, REWARDS_BY_ID -from nte_history_exporter.decoder.protocol import decode_reward_key, infer_reward_type -from nte_history_exporter.decoder.user_uid import extract_user_uid -from nte_history_exporter.export.csv_export import write_csv -from nte_history_exporter.decoder.arc import ( - arc_request_page, - build_arc_rows_from_pairs, - decode_arc_key, - decode_arc_timestamp, - is_arc_history_request, - make_arc_uid, - parse_arc_response, -) -from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket -from nte_history_exporter.live_capture.libpcap import ( - DLT_EN10MB, - DLT_LINUX_SLL, - DLT_LINUX_SLL2, - DLT_LOOP, - DLT_RAW, - _extract_ipv4_frame, - _load_library, - LibpcapUnavailable, -) -from nte_history_exporter.live_capture.windows_raw import parse_ipv4_packet -from nte_history_exporter.live_capture.backends import open_capture_backend -from nte_history_exporter.export.json_export import build_export_json -from nte_history_exporter.live_capture.runner import export_paths -from nte_history_exporter.pool_mappings import load_pool_mappings, pool_meta_from_mapping -from nte_history_exporter.update_check import UpdateInfo, check_for_update, is_newer_version - - -def load_reference_csv(name): - path = EXPORTS / name - if not path.exists(): - path = EXPORTS / "old" / name - if not path.exists(): - raise unittest.SkipTest(f"private reference fixture not present: {path}") - rows = [] - with path.open(newline="", encoding="utf-8") as f: - for row in csv.DictReader(f): - normalized = dict(row) - for key in ["page", "offset", "row", "dice", "dice_raw_u32", "quantity"]: - if normalized.get(key) not in ("", None): - normalized[key] = int(normalized[key]) - rows.append(normalized) - return rows - - -def decode_single_record(record_hex): - return decode_response_records(bytes(0x50) + bytes.fromhex(record_hex))[0] - - -def load_v7_row(name, one_based_index): - path = V7_EXPORTS / name - if not path.exists(): - raise unittest.SkipTest(f"private reference fixture not present: {path}") - with path.open(newline="", encoding="utf-8") as f: - rows = list(csv.DictReader(f)) - return rows[one_based_index - 1] - - -def load_arc_csv(name): - path = ARC_EXPORTS / name - if not path.exists(): - raise unittest.SkipTest(f"private reference fixture not present: {path}") - with path.open(newline="", encoding="utf-8") as f: - return list(csv.DictReader(f)) - - -class BoundaryExportTests(unittest.TestCase): - def test_windows_auto_prefers_npcap(self): - capture = Mock(device=r"\Device\NPF_test") - with ( - patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"), - patch( - "nte_history_exporter.live_capture.backends.open_libpcap_capture", - return_value=capture, - ), - patch("nte_history_exporter.live_capture.backends.RawSocketCapture") as raw_capture, - ): - selected = open_capture_backend("192.0.2.1") - - self.assertIs(selected, capture) - self.assertEqual(selected.name, "npcap") - self.assertEqual(selected.fallback_reason, "") - raw_capture.assert_not_called() - - def test_windows_auto_falls_back_to_raw_socket_without_npcap(self): - fallback = Mock() - with ( - patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"), - patch( - "nte_history_exporter.live_capture.backends.open_libpcap_capture", - side_effect=LibpcapUnavailable("Npcap could not be loaded"), - ), - patch( - "nte_history_exporter.live_capture.backends.RawSocketCapture", - return_value=fallback, - ) as raw_capture, - ): - selected = open_capture_backend("192.0.2.1") - - self.assertIs(selected, fallback) - raw_capture.assert_called_once_with( - "192.0.2.1", - fallback_reason="Npcap could not be loaded", - ) - - def test_explicit_libpcap_does_not_fall_back(self): - with ( - patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"), - patch( - "nte_history_exporter.live_capture.backends.open_libpcap_capture", - side_effect=LibpcapUnavailable("Npcap could not be loaded"), - ), - patch("nte_history_exporter.live_capture.backends.RawSocketCapture") as raw_capture, - self.assertRaises(LibpcapUnavailable), - ): - open_capture_backend("192.0.2.1", "libpcap") - - raw_capture.assert_not_called() - - def test_windows_libpcap_load_prefers_system_then_path(self): - system_dir = Path("C:/Windows/System32/Npcap") - loaded = Mock() - with ( - patch("nte_history_exporter.live_capture.libpcap.sys.platform", "win32"), - patch("nte_history_exporter.live_capture.libpcap._windows_npcap_directory", return_value=system_dir), - patch("nte_history_exporter.live_capture.libpcap.Path.is_dir", return_value=True), - patch("nte_history_exporter.live_capture.libpcap.os.add_dll_directory", return_value=Mock(), create=True), - patch( - "nte_history_exporter.live_capture.libpcap.ctypes.CDLL", - side_effect=[OSError("system missing"), loaded], - ) as cdll, - ): - self.assertIs(_load_library(), loaded) - - self.assertEqual( - [call.args[0] for call in cdll.call_args_list], - [ - str(system_dir / "wpcap.dll"), - "wpcap.dll", - ], - ) - - def test_libpcap_link_layers_extract_ipv4_packets(self): - ip_packet = bytes.fromhex("4500001c0000000040110000c0000201c6336402") + bytes(8) - ethernet = bytes(12) + bytes.fromhex("0800") + ip_packet - loop = (2).to_bytes(4, sys.byteorder) + ip_packet - linux_sll = bytes(14) + bytes.fromhex("0800") + ip_packet - linux_sll2 = bytes.fromhex("0800") + bytes(18) + ip_packet - - self.assertEqual(_extract_ipv4_frame(ethernet, DLT_EN10MB), ip_packet) - self.assertEqual(_extract_ipv4_frame(ip_packet, DLT_RAW), ip_packet) - self.assertEqual(_extract_ipv4_frame(loop, DLT_LOOP), ip_packet) - self.assertEqual(_extract_ipv4_frame(linux_sll, DLT_LINUX_SLL), ip_packet) - self.assertEqual(_extract_ipv4_frame(linux_sll2, DLT_LINUX_SLL2), ip_packet) - - def test_pool_mapping_json_files_have_uniform_shape(self): - required_top_level = {"pool_key", "game", "system", "banner", "request", "response"} - for pool_key, mapping in load_pool_mappings().items(): - with self.subTest(pool_key=pool_key): - self.assertEqual(set(required_top_level) - set(mapping), set()) - self.assertEqual(mapping["pool_key"], pool_key) - self.assertIn("id", mapping["system"]) - self.assertIn("name", mapping["system"]) - self.assertIn("id", mapping["banner"]) - self.assertIn("name", mapping["banner"]) - self.assertIn("shared_pity", mapping["banner"]) - self.assertIn("family", mapping["request"]) - self.assertIn("length", mapping["request"]) - self.assertIn("constant", mapping["request"]) - self.assertIn("cursor_step", mapping["request"]) - - def test_pool_mapping_json_matches_runtime_pool_meta(self): - for pool_key, mapping in load_pool_mappings().items(): - with self.subTest(pool_key=pool_key): - self.assertEqual(pool_meta_from_mapping(mapping), POOL_META[pool_key]) - - def test_update_version_comparison_handles_release_tags(self): - self.assertTrue(is_newer_version("v0.1.7", "0.1.6")) - self.assertTrue(is_newer_version("0.2.0", "0.1.6")) - self.assertTrue(is_newer_version("v0.1.10", "0.1.9")) - self.assertFalse(is_newer_version("v0.1.6", "0.1.6")) - self.assertFalse(is_newer_version("v0.1.5", "0.1.6")) - self.assertFalse(is_newer_version("latest", "0.1.6")) - - def test_update_check_reports_newer_github_release(self): - latest = { - "tag_name": "v0.1.7", - "html_url": "https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.7", - } - with patch("nte_history_exporter.update_check.fetch_latest_release", return_value=latest): - update = check_for_update("0.1.6", timeout=0.1) - - self.assertEqual( - update, - UpdateInfo( - current_version="0.1.6", - latest_version="v0.1.7", - release_url="https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.7", - ), - ) - - def test_update_check_ignores_prerelease(self): - latest = { - "tag_name": "v0.1.8-dev-branch.123", - "html_url": "https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.8-dev-branch.123", - "prerelease": True, - } - with patch("nte_history_exporter.update_check.fetch_latest_release", return_value=latest): - self.assertIsNone(check_for_update("0.1.7", timeout=0.1)) - - def test_update_check_is_quiet_when_unavailable_or_current(self): - with patch("nte_history_exporter.update_check.fetch_latest_release", side_effect=OSError("offline")): - self.assertIsNone(check_for_update("0.1.6", timeout=0.1)) - - with patch("nte_history_exporter.update_check.fetch_latest_release", return_value={"tag_name": "v0.1.6"}): - self.assertIsNone(check_for_update("0.1.6", timeout=0.1)) - - def test_reward_mapping_files_have_expected_shape(self): - self.assertTrue(ARC_META) - for arc_id, meta in ARC_META.items(): - with self.subTest(arc_id=arc_id): - self.assertTrue(arc_id.startswith("fork_")) - self.assertIn("name", meta) - self.assertIn(meta.get("rank"), ("S", "A", "B")) - - self.assertTrue(CHARACTERS) - for character_id, info in CHARACTERS.items(): - with self.subTest(character_id=character_id): - self.assertTrue(character_id.isdigit()) - self.assertIn("name", info) - self.assertIn(info.get("rank"), ("S", "A")) - - self.assertTrue(ITEMS) - for item_id, info in ITEMS.items(): - with self.subTest(item_id=item_id): - self.assertIn(info.get("type"), ("item", "cosmetic")) - self.assertIn("name", info) - - def test_rewards_by_id_merges_all_mapping_files(self): - for reward_id in (*ARC_META, *CHARACTERS, *ITEMS): - with self.subTest(reward_id=reward_id): - reward = REWARDS_BY_ID[reward_id] - self.assertEqual(reward["id"], reward_id) - self.assertIn(reward["type"], ("arc", "character", "item", "cosmetic")) - - def test_decode_reward_key_round_trips_observed_keys(self): - observed = { - "98bdc9ad7dd9a5b99501": "fork_vine", - "98bdc9ad7d41c9bdad85c9e5bdb901": "fork_Prokaryon", - "98bdc9ad7ddda1d585add585b99d01": "fork_whuakuang", - "98bdc9ad7dddd5a1d585add585b99d01": "fork_wuhuakuang", - "10a58d9539bdc9b585b101": "DiceNormal", - "10a58d957dd1a58dad95d17dc1c400": "Dice_ticket_01", - "10a58d957dd1a58dad95d17dc1c800": "Dice_ticket_02", - "10a58d95b1a5b5a5d19501": "Dicelimite", - "1885cda1a5bdb97d1db1a591957dc5c0c4c000": "Fashion_Glide_1010", - "1885cda1a5bdb97dd995a1a58db1957dc5c0c4c07c59c1c0e000": "Fashion_vehicle_1010_V008", - "c4c0cccc00": "1033", - "c4c0dcc000": "1070", - "c4c0dcc0": "1070", - "c4c0c8c4": "1021", - } - for key_hex, expected_id in observed.items(): - with self.subTest(key_hex=key_hex): - self.assertEqual(decode_reward_key(bytes.fromhex(key_hex)), expected_id) - - def test_infer_reward_type_for_unmapped_ids(self): - self.assertEqual(infer_reward_type("fork_newarc"), "arc") - self.assertEqual(infer_reward_type("1099"), "character") - self.assertEqual(infer_reward_type("Fashion_hat_2000"), "cosmetic") - self.assertEqual(infer_reward_type("Dice_ticket_03"), "item") - self.assertEqual(infer_reward_type(""), "") - - def test_uid_source_matches_v4_reference(self): - rows = load_reference_csv("monopoly_history_poc_10_all_44_pages_v4.csv") - first = rows[0] - self.assertEqual(make_uid(first, 0), "f2c72f0a80b79216bf15661521620693") - - def test_uid_uses_pool_timestamp_and_ordinal_only(self): - row = { - "pool_group_id": "Lottery_LimitedCharacter", - "timestamp_raw_hex": "40e93247c3097b23", - "dice": 5, - "reward_key_hex": "10a58d957dd1a58dad95d17dc1c800", - "quantity": 50, - } - changed_content = { - **row, - "dice": 1, - "reward_key_hex": "98bdc9ad7dd9a5b99501", - "quantity": 1, - } - changed_pool = {**row, "pool_group_id": "Lottery_Permanent"} - - self.assertEqual(make_uid(row, 0), "74a9ef4aacde549dfe8e8e7cc6ddd65b") - self.assertEqual(make_uid(changed_content, 0), make_uid(row, 0)) - self.assertNotEqual(make_uid(changed_pool, 0), make_uid(row, 0)) - self.assertNotEqual(make_uid(row, 1), make_uid(row, 0)) - - def test_pages_1_to_5_exports_every_row(self): - rows = load_reference_csv("monopoly_history_poc_13_pages_1_to_5_v4.csv") - annotated = annotate_groups(rows) - - exported = [row for row in annotated if row["export_record"] is True] - - # Every decoded row is exported; boundary groups are never dropped. - self.assertEqual(len(annotated), 25) - self.assertEqual(len(exported), 25) - - @staticmethod - def _synthetic_row(page, timestamp_hex, result_type): - return { - "page": page, - "timestamp_raw_hex": timestamp_hex, - "timestamp_decoded": f"ts-{timestamp_hex}", - "result_type": result_type, - "dice": 4 if result_type == "dice" else 0, - "reward_key_hex": "10a58d9539bdc9b585b101", - "quantity": 1, - } - - def test_oldest_group_with_partial_dice_count_exports_without_warning(self): - rows = [self._synthetic_row(1, "aa", "dice") for _ in range(5)] - rows += [self._synthetic_row(2, "bb", "dice") for _ in range(5)] - rows += [self._synthetic_row(3, "bb", "dice") for _ in range(4)] - rows += [self._synthetic_row(3, "bb", "points_gift")] - - annotated = annotate_groups(rows) - exported = [row for row in annotated if row["export_record"] is True] - - # Oldest group is a partially captured 10-pull on a full final page. Its - # captured prefix is ordinal-stable, so it is exported with stable UIDs. - self.assertEqual(len(exported), 15) - oldest = [row for row in annotated if row["timestamp_raw_hex"] == "bb"] - self.assertTrue(all(row["uid"] for row in oldest)) - self.assertTrue(all(row["uid_status"] == "stable" for row in oldest)) - self.assertEqual([row["timestamp_group_ordinal"] for row in oldest], list(range(10))) - - def test_incomplete_oldest_prefix_keeps_stable_uids(self): - full = [self._synthetic_row(1, "aa", "dice") for _ in range(5)] - full += [self._synthetic_row(2, "bb", "dice") for _ in range(3)] - full += [self._synthetic_row(3, "bb", "dice") for _ in range(2)] - truncated = [r for r in full if r["page"] in (1, 2)] - - full_rows = annotate_groups([dict(r) for r in full]) - trunc_rows = annotate_groups([dict(r) for r in truncated]) - - full_uids = [r["uid"] for r in full_rows if r["timestamp_raw_hex"] == "bb"][:3] - trunc_uids = [r["uid"] for r in trunc_rows if r["timestamp_raw_hex"] == "bb"] - # Capturing only the first 3 of a 5-record oldest group yields the same - # UIDs those rows have in the full capture. - self.assertEqual(len(trunc_uids), 3) - self.assertEqual(trunc_uids, full_uids) - - def test_oldest_group_with_ten_dice_exports_on_full_final_page(self): - rows = [self._synthetic_row(1, "aa", "dice") for _ in range(5)] - rows += [self._synthetic_row(2, "bb", "dice") for _ in range(5)] - rows += [self._synthetic_row(3, "bb", "dice") for _ in range(5)] - - annotated = annotate_groups(rows) - exported = [row for row in annotated if row["export_record"] is True] - - self.assertEqual(len(exported), 15) - - def test_run_selection_anchors_to_page_1_and_keeps_newest(self): - # Page 2's response was lost: captured pages 1, 3, 4, 5. - pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (1, 3, 4, 5)] - run, warnings = select_continuous_run_from_page_1(pairs) - - # The page-1 run (just page 1, the newest history) is kept; later pages are - # ignored with a gap warning, never silently discarding page 1. - self.assertEqual([p[0] for p in run], [1]) - self.assertEqual(len(warnings), 1) - self.assertEqual(warnings[0]["code"], "PAGE_GAP_DETECTED") - self.assertEqual(warnings[0]["ignored_pages"], [3, 4, 5]) - - def test_run_selection_warns_when_page_1_missing(self): - pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (3, 4, 5)] - run, warnings = select_continuous_run_from_page_1(pairs) - - self.assertEqual([p[0] for p in run], [3, 4, 5]) - self.assertEqual(warnings[0]["code"], "DID_NOT_START_AT_PAGE_1") - - def test_full_reference_scan_exports_all_rows(self): - rows = load_reference_csv("monopoly_history_poc_10_all_44_pages_v4.csv") - annotated = annotate_groups(rows) - exported = [row for row in annotated if row["export_record"] is True] - - json_path = EXPORTS / "monopoly_history_export_10_all_44_pages_v4.json" - if not json_path.exists(): - json_path = EXPORTS / "old" / "monopoly_history_export_10_all_44_pages_v4.json" - if not json_path.exists(): - raise unittest.SkipTest(f"private reference fixture not present: {json_path}") - with json_path.open(encoding="utf-8") as f: - reference = json.load(f) - - self.assertEqual(len(annotated), reference["scan"]["decoded_records"]) - self.assertEqual(len(exported), reference["scan"]["exported_records"]) - - def test_sanitized_export_omits_raw_packet_fields(self): - rows = load_reference_csv("monopoly_history_poc_13_pages_1_to_5_v4.csv") - annotated = annotate_groups(rows) - export = build_export_json(annotated, []) - - self.assertEqual(export["format"], "nte-history-export") - self.assertIn("exporter", export) - self.assertNotIn("user_uid", export) - self.assertNotIn("record_hex", export["records"][0]) - self.assertNotIn("request_msg", export["records"][0]) - self.assertNotIn("response_msg", export["records"][0]) - - def test_export_includes_user_uid_when_provided(self): - rows = load_reference_csv("monopoly_history_poc_13_pages_1_to_5_v4.csv") - annotated = annotate_groups(rows) - export = build_export_json( - annotated, - [], - capture_source="npcap", - user_uid="123456789", - ) - - self.assertEqual(list(export).index("user_uid"), list(export).index("records") - 1) - self.assertEqual(export["capture_source"], "npcap") - self.assertEqual(export["user_uid"], "123456789") - - def test_debug_csv_includes_exporter_version(self): - with TemporaryDirectory() as tmp: - path = Path(tmp) / "debug.csv" - write_csv(path, [{"uid": "abc123"}]) - - with path.open(newline="", encoding="utf-8") as f: - rows = list(csv.DictReader(f)) - - self.assertEqual(rows[0]["exporter_version"], __version__) - self.assertEqual(rows[0]["uid"], "abc123") - - def test_export_paths_include_user_uid_banner_and_timestamp(self): - _csv_path, json_path = export_paths("limited_character", "218216016349") - - self.assertRegex( - json_path.name, - r"^218216016349_Limited_\d{8}_\d{6}(?:_\d+)?\.json$", - ) - - def test_extracts_user_uid_from_record_context(self): - payload = ( - b"\x00" * 24 - + (218216016349).to_bytes(8, "little") - + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00" - ) - - self.assertEqual(extract_user_uid(payload), "218216016349") - - def test_extracts_user_uid_from_private_spawn_record_context(self): - payload = ( - b"\x88\x00\x00\x00\x10\x00\x00\x00" - + (218216016349).to_bytes(8, "little") - + b"\x08\x00\x0c\x00\x07\x00\x08\x00\x08\x00\x00\x00" - + b"\x00\x00\x00\x01\x08\x00\x00\x00\x04\x00\x04\x00" - + b"\x04\x00\x00\x00\x16\x00\x00\x00PrivateSpawnInfoRecord\x00" - ) - - self.assertEqual(extract_user_uid(payload), "218216016349") - - def test_does_not_extract_user_uid_from_wrong_record_offset(self): - payload = ( - b"\x00" * 28 - + (218216016349).to_bytes(8, "little") - + b"\x00\x00\x00\x00TagOthers\x00" - ) - - self.assertIsNone(extract_user_uid(payload)) - - def test_does_not_extract_old_eight_digit_false_positive_as_user_uid(self): - payload = ( - b"WholeVehicleData\x00\x00\x00\x00\x00o<\x00\x00\x05\x00\x00\x00" - b"\x0b\x00\x00\x00Vehicle015\x00\x0b\x00\x00\x00buyvehicle\x00" - b"\x09\x00\x00\x0015363624\x00\x06\x00\x00\x00" - ) - - self.assertIsNone(extract_user_uid(payload)) - - def test_ipv4_parser_extracts_tcp_payload_for_user_uid_detection(self): - payload = ( - (218216016349).to_bytes(8, "little") - + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00" - ) - tcp_header = bytearray(20) - tcp_header[0:2] = (40000).to_bytes(2, "big") - tcp_header[2:4] = (30000).to_bytes(2, "big") - tcp_header[12] = 5 << 4 - total_len = 20 + len(tcp_header) + len(payload) - ip_header = bytearray(20) - ip_header[0] = 0x45 - ip_header[2:4] = total_len.to_bytes(2, "big") - ip_header[9] = 6 - ip_header[12:16] = bytes([192, 0, 2, 1]) - ip_header[16:20] = bytes([198, 51, 100, 2]) - - packet = parse_ipv4_packet(bytes(ip_header) + bytes(tcp_header) + payload) - - self.assertIsNotNone(packet) - self.assertEqual(packet.protocol, "tcp") - self.assertEqual(packet.payload, payload) - - def test_limited_selector_and_marker_decode(self): - request = bytearray(45) - request[31:35] = (4).to_bytes(4, "little") - request[35:39] = (4220).to_bytes(4, "little") - request[40:44] = (8).to_bytes(4, "little") - self.assertEqual(history_request_kind(bytes(request)), "limited_character") - - def test_monopoly_request_allows_coalesced_trailing_payload(self): - request = bytearray(45) - request[31:35] = (25 * 4).to_bytes(4, "little") - request[35:39] = (4220).to_bytes(4, "little") - request[40:44] = (4).to_bytes(4, "little") - coalesced = bytes(request) + bytes.fromhex( - "007c669610062038461bc40100000872a34b93821a0219aa933b0a6b2ba34a6b" - ) - - self.assertEqual(history_request_kind(coalesced), "permanent") - - def test_arc_request_allows_coalesced_trailing_payload(self): - request = bytearray(34) - request[24:28] = (2060).to_bytes(4, "little") - request[29:33] = (7 * 2).to_bytes(4, "little") - - self.assertTrue(is_arc_history_request(bytes(request) + bytes(32))) - self.assertEqual(arc_request_page(bytes(request) + bytes(32)), 7) - - response = bytearray(220) - response[0x50:0x54] = (4).to_bytes(4, "little") - response[0x54:0x58] = (20).to_bytes(4, "little") - response[0x58:0x5d] = bytes.fromhex("c4c0c4c000") - marker_offset = 0x5d - response[marker_offset:marker_offset + len(LIMITED_CHARACTER_MARKER)] = LIMITED_CHARACTER_MARKER - timestamp_raw = (2556647947780680000).to_bytes(8, "little") - response[marker_offset + len(LIMITED_CHARACTER_MARKER):marker_offset + len(LIMITED_CHARACTER_MARKER) + 8] = timestamp_raw - - rows = decode_response_records(bytes(response)) - self.assertEqual(len(rows), 1) - self.assertEqual(rows[0]["reward_id"], "1010") - self.assertEqual(rows[0]["reward_name"], "Nanally") - - def test_v7_prefixed_points_gift_rows_override_visible_dice(self): - cases = [ - ("limited_all_04_v7.csv", 41, "1020"), - ("monopoly_history_poc_10_all_44_pages_v7.csv", 31, "1033"), - ("monopoly_history_poc_10_all_44_pages_v7.csv", 86, "fork_PaperPlane"), - ("monopoly_history_poc_10_all_44_pages_v7.csv", 141, "fork_Kite"), - ("monopoly_history_poc_10_all_44_pages_v7.csv", 196, "1021"), - ] - for filename, row_index, reward_id in cases: - with self.subTest(filename=filename, row_index=row_index): - reference = load_v7_row(filename, row_index) - decoded = decode_single_record(reference["record_hex"]) - self.assertEqual(decoded["result_type"], "points_gift") - self.assertEqual(decoded["result_source_raw"], 0) - self.assertEqual(decoded["dice"], 0) - self.assertEqual(decoded["dice_raw_u32"], 0) - self.assertEqual(decoded["reward_id"], reward_id) - - def test_v7_prefixed_chase_reward_overrides_visible_dice_and_quantity(self): - reference = load_v7_row("limited_all_04_v7.csv", 61) - decoded = decode_single_record(reference["record_hex"]) - decoded.update( - { - "pool_group_id": "Lottery_LimitedCharacter", - "timestamp_group_ordinal": int(reference["timestamp_group_ordinal"]), - } - ) - - self.assertEqual(decoded["result_type"], "chase_reward") - self.assertEqual(decoded["result_source_raw"], -4) - self.assertEqual(decoded["dice"], -4) - self.assertEqual(decoded["dice_raw_u32"], -4) - self.assertEqual(decoded["reward_id"], "Dice_ticket_01") - self.assertEqual(decoded["quantity"], 30) - self.assertEqual(make_uid(decoded, int(reference["timestamp_group_ordinal"])), "23dc293f39f18e94a81ccaaf7e1a67eb") - - def test_warp_piece_chase_subrecord_without_prefix_marker_is_chase_reward(self): - decoded = decode_single_record( - "c1c4b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c400" - "4c0000000c85c99141bdbdb17d0da185c9858dd195c90140eb2c2dd7227b23" - ) - - self.assertEqual(decoded["result_type"], "chase_reward") - self.assertEqual(decoded["result_source_raw"], -4) - self.assertEqual(decoded["dice"], -4) - self.assertEqual(decoded["dice_raw_u32"], -4) - self.assertEqual(decoded["reward_id"], "Dice_ticket_01") - self.assertEqual(decoded["reward_name"], "Warp Piece") - self.assertEqual(decoded["quantity"], 30) - - def test_page_first_prefix_uses_real_dice_field(self): - cases = [ - ( - "003006000014000000040000002800000098bdc9ad7dd9a5b995010000000008000000" - "3c00000010a58d957dd1a58dad95d17dc1c4002800000098bdc9ad7dd9a5b995014c" - "0000000c85c99141bdbdb17d0da185c9858dd195c901c0dd53bd2b137b23", - 1, - "fork_vine", - ), - ( - "00c8060000140000001000000014000000c4c0d4d400000000000400000014000000" - "c4c0d4d400440000000c85c99141bdbdb17d3995dd49bdb1950100d929e115087b23", - 4, - "1055", - ), - ] - for record_hex, expected_dice, reward_id in cases: - with self.subTest(reward_id=reward_id): - decoded = decode_single_record(record_hex) - - self.assertEqual(decoded["dice"], expected_dice) - self.assertEqual(decoded["dice_raw_u32"], expected_dice * 4) - self.assertEqual(decoded["dice_offset_in_record"], 9) - self.assertEqual(decoded["result_type"], "dice") - self.assertEqual(decoded["reward_id"], reward_id) - - def test_batched_monopoly_response_normalizes_embedded_page_header(self): - page_7 = [load_v7_row("limited_all_04_v7.csv", row) for row in range(31, 36)] - page_8 = [load_v7_row("limited_all_04_v7.csv", row) for row in range(36, 41)] - embedded_header = bytes.fromhex( - "100126675671c6e0ecfcb769070000000c08000000100000003e0800000004000000" - "486c0000001835bdb9bdc1bdb1e531bdd1d195c9e549958dbdc9911185d18501" - ) - response = ( - bytes(0x50) - + b"".join(bytes.fromhex(row["record_hex"]) for row in page_7) - + embedded_header - + b"".join(bytes.fromhex(row["record_hex"]) for row in page_8) - ) - - decoded = decode_response_records(response) - - self.assertEqual(len(decoded), 10) - self.assertEqual(decoded[5]["reward_id"], page_8[0]["reward_id"]) - self.assertEqual(decoded[5]["dice"], 4) - self.assertEqual(decoded[5]["result_type"], "dice") - self.assertEqual(decoded[5]["record_hex"], page_8[0]["record_hex"]) - - def test_monopoly_response_parser_realigns_bit_packed_payload(self): - reference_rows = [ - load_v7_row("limited_all_04_v7.csv", row) - for row in range(51, 61) - ] - response = bytes(0x50) + b"".join( - bytes.fromhex(row["record_hex"]) for row in reference_rows - ) - packed = bytearray() - carry = 0b10101 - for byte in response: - packed.append(carry | ((byte << 5) & 0xFF)) - carry = byte >> 3 - packed.append(carry) - - decoded = decode_response_records(bytes(packed)) - - self.assertEqual(len(decoded), 10) - self.assertEqual( - [row["record_hex"] for row in decoded], - [row["record_hex"] for row in reference_rows], - ) - - def test_page_gap_warning_reports_ignored_pages(self): - pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (1, 2, 3, 5)] - run, warnings = select_continuous_run_from_page_1(pairs) - self.assertEqual([p[0] for p in run], [1, 2, 3]) - self.assertEqual(warnings[0]["code"], "PAGE_GAP_DETECTED") - self.assertEqual(warnings[0]["ignored_pages"], [5]) - - def test_arc_key_timestamp_and_uid_match_reference(self): - row = load_arc_csv("arc_pull_10_all_pages_v2.csv")[0] - self.assertEqual(decode_arc_key(bytes.fromhex(row["arc_key_hex"])), "fork_nonos") - _ticks, _unix, decoded = decode_arc_timestamp(bytes.fromhex(row["timestamp_raw_hex"])) - self.assertEqual(decoded, "2026-06-10 23:46:29") - self.assertEqual(make_arc_uid(row["timestamp_raw_hex"], int(row["timestamp_group_ordinal"])), "4435d9729fa8fd0eaf1b1ad7aa4d2172") - - def test_arc_response_parser_matches_reference_first_page(self): - reference_rows = load_arc_csv("arc_pull_10_all_pages_v2.csv")[:5] - response = bytes(0x4C) + b"".join(bytes.fromhex(row["record_hex"]) for row in reference_rows) - decoded = parse_arc_response(response) - self.assertEqual(len(decoded), 5) - self.assertEqual([row["reward_id"] for row in decoded[:3]], ["fork_nonos", "fork_nonos", "fork_Prokaryon"]) - self.assertEqual(decoded[0]["reward_type"], "arc") - self.assertEqual(decoded[0]["reward_key_hex"], reference_rows[0]["arc_key_hex"]) - - def test_arc_response_parser_rejects_invalid_timestamp_noise(self): - response = bytearray(0x4C) - response += (10).to_bytes(4, "little") - response += bytes.fromhex("ccdee4d6be") - response += (8).to_bytes(4, "little") - response += b"garb" - response += (0xFFFFFFFFFFFFFFFF).to_bytes(8, "little") - - self.assertEqual(parse_arc_response(bytes(response)), []) - - def test_arc_partial_timestamp_group_is_exported_without_warning(self): - rows = load_arc_csv("arc_pages_1_to_5_v2.csv") - pairs = [] - for page in range(1, 6): - page_rows = [row for row in rows if int(row["page"]) == page] - response = bytes(0x4C) + b"".join(bytes.fromhex(row["record_hex"]) for row in page_rows) - pairs.append((page, page * 2, page, 1.0, page + 100, 1.1, response)) - decoded = build_arc_rows_from_pairs(pairs) - exported = [row for row in decoded if row["export_record"] is True] - - # The oldest group is a 10-pull split by stopping at page 5 (5 of 10 rows). - # Its captured prefix is ordinal-stable, so every row is exported with a - # stable UID and no warning. - self.assertEqual(len(decoded), 25) - self.assertEqual(len(exported), 25) - self.assertTrue(all(row["uid"] for row in decoded)) - self.assertTrue(all(row["uid_status"] == "stable" for row in decoded)) - - def test_arc_incomplete_prefix_keeps_stable_uids(self): - rows = load_arc_csv("arc_pull_10_all_pages_v2.csv") - # Build a full 2-page (10-record) group, then a truncated 1-page version. - full_pairs, trunc_pairs = [], [] - for page in (1, 2): - page_rows = [r for r in rows if int(r["page"]) == page] - response = bytes(0x4C) + b"".join(bytes.fromhex(r["record_hex"]) for r in page_rows) - full_pairs.append((page, page * 2, page, 1.0, page + 100, 1.1, response)) - if page == 1: - trunc_pairs.append((page, page * 2, page, 1.0, page + 100, 1.1, response)) - - ts = rows[0]["timestamp_raw_hex"] - full = [r for r in build_arc_rows_from_pairs(full_pairs) if r["timestamp_raw_hex"] == ts] - trunc = [r for r in build_arc_rows_from_pairs(trunc_pairs) if r["timestamp_raw_hex"] == ts] - self.assertTrue(trunc) - self.assertEqual([r["uid"] for r in trunc], [r["uid"] for r in full[: len(trunc)]]) - - def test_arc_row_builder_accepts_live_pairs_with_kind(self): - reference_rows = load_arc_csv("arc_pull_10_all_pages_v2.csv")[:5] - response = bytes(0x4C) + b"".join(bytes.fromhex(row["record_hex"]) for row in reference_rows) - decoded = build_arc_rows_from_pairs([(1, 2, 1, 1.0, 2, 1.1, response, "arc_miracle_box")]) - self.assertEqual(len(decoded), 5) - self.assertEqual(decoded[0]["reward_id"], "fork_nonos") - - def test_arc_export_is_shared_pity(self): - reference_rows = load_arc_csv("arc_pull_10_all_pages_v2.csv")[:10] - response = bytes(0x4C) + b"".join(bytes.fromhex(row["record_hex"]) for row in reference_rows) - rows = build_arc_rows_from_pairs([(1, 2, 1, 1.0, 2, 1.1, response, "arc_miracle_box")]) - export = build_export_json(rows, []) - self.assertEqual(export["banner"]["id"], "Arc_MiracleBox") - self.assertIs(export["banner"]["shared_pity"], True) - - def test_group_detection_counts_only_dice_records_but_uid_ordinals_keep_all_rows(self): - rows = [ - { - "page": 1, - "timestamp_raw_hex": "aa", - "timestamp_decoded": "2026-01-01 00:00:00", - "result_type": "dice", - "dice": 1, - "reward_key_hex": "k1", - "quantity": 1, - }, - { - "page": 1, - "timestamp_raw_hex": "aa", - "timestamp_decoded": "2026-01-01 00:00:00", - "result_type": "points_gift", - "dice": 0, - "reward_key_hex": "k2", - "quantity": 1, - }, - { - "page": 1, - "timestamp_raw_hex": "aa", - "timestamp_decoded": "2026-01-01 00:00:00", - "result_type": "chase_reward", - "dice": -4, - "reward_key_hex": "k3", - "quantity": 30, - }, - { - "page": 1, - "timestamp_raw_hex": "aa", - "timestamp_decoded": "2026-01-01 00:00:00", - "result_type": "dice", - "dice": 2, - "reward_key_hex": "k4", - "quantity": 1, - }, - { - "page": 1, - "timestamp_raw_hex": "bb", - "timestamp_decoded": "2026-01-01 00:01:00", - "result_type": "dice", - "dice": 3, - "reward_key_hex": "k5", - "quantity": 1, - }, - ] - - annotated = annotate_groups(rows) - - # Ordinals cover every row in the group, but the dice-only count drives - # timestamp_group_size_seen (2 dice in the 4-record group). - self.assertEqual([row["timestamp_group_ordinal"] for row in annotated[:4]], [0, 1, 2, 3]) - self.assertEqual({row["timestamp_group_size_seen"] for row in annotated[:4]}, {2}) - self.assertEqual({row["timestamp_group_record_size_seen"] for row in annotated[:4]}, {4}) - self.assertTrue(all(row["export_record"] for row in annotated)) - - def test_live_session_pairs_request_and_response(self): - session = LiveHistorySession("192.168.0.10") - - request = bytearray(45) - request[31:35] = (4).to_bytes(4, "little") - request[35:39] = (4220).to_bytes(4, "little") - request[40:44] = (4).to_bytes(4, "little") - - response = bytearray(220) - response[0x50:0x50 + len(MARKER)] = MARKER - response[0x50 + len(MARKER):0x50 + len(MARKER) + 8] = ( - 2556647947780680000 - ).to_bytes(8, "little") - - self.assertFalse( - session.process_packet( - UdpPacket( - timestamp=1.0, - src_ip="192.168.0.10", - dst_ip="203.0.113.5", - src_port=50000, - dst_port=40000, - payload=bytes(request), - ) - ) - ) - self.assertTrue( - session.process_packet( - UdpPacket( - timestamp=1.2, - src_ip="203.0.113.5", - dst_ip="192.168.0.10", - src_port=40000, - dst_port=50000, - payload=bytes(response), - ) - ) - ) - self.assertEqual(len(session.pairs), 1) - self.assertEqual(session.last_page_seen, 1) - - def test_live_session_pairs_pipelined_and_batched_pages(self): - session = LiveHistorySession("192.168.0.10") - - def request_packet(page, timestamp): - request = bytearray(45) - request[31:35] = (page * 4).to_bytes(4, "little") - request[35:39] = (4220).to_bytes(4, "little") - request[40:44] = (4).to_bytes(4, "little") - return UdpPacket( - timestamp=timestamp, - src_ip="192.168.0.10", - dst_ip="203.0.113.5", - src_port=50000, - dst_port=40000, - payload=bytes(request), - ) - - timestamp = (2556647947780680000).to_bytes(8, "little") - - def response_packet(record_count, packet_timestamp): - response = bytearray(0x50) - for _ in range(record_count): - response += bytes(4) + MARKER + timestamp - return UdpPacket( - timestamp=packet_timestamp, - src_ip="203.0.113.5", - dst_ip="192.168.0.10", - src_port=40000, - dst_port=50000, - payload=bytes(response), - ) - - for page in range(1, 9): - session.process_packet(request_packet(page, 1.0 + page / 10)) - - self.assertTrue(session.process_packet(response_packet(10, 2.0))) - self.assertEqual([pair[0] for pair in session.pairs], [1, 2]) - self.assertEqual([pair[8:10] for pair in session.pairs], [(0, 5), (5, 5)]) - - self.assertTrue(session.process_packet(response_packet(10, 2.1))) - self.assertEqual([pair[0] for pair in session.pairs], [1, 2, 3, 4]) - self.assertEqual(session.missing_pages("permanent"), []) - rows = session.build_rows("permanent") - self.assertEqual(len(rows), 20) - self.assertEqual( - {page: sum(row["page"] == page for row in rows) for page in range(1, 5)}, - {1: 5, 2: 5, 3: 5, 4: 5}, - ) - - session.process_packet(request_packet(9, 2.2)) - self.assertTrue(session.process_packet(response_packet(4, 2.3))) - self.assertEqual(session.pairs[-1][0], 9) - self.assertEqual(session.pairs[-1][8:10], (0, 4)) - - def test_live_session_new_page_one_starts_clean_recovery_cycle(self): - session = LiveHistorySession("192.168.0.10") - - def request_packet(page, timestamp): - request = bytearray(45) - request[31:35] = (page * 4).to_bytes(4, "little") - request[35:39] = (4220).to_bytes(4, "little") - request[40:44] = (4).to_bytes(4, "little") - return UdpPacket( - timestamp=timestamp, - src_ip="192.168.0.10", - dst_ip="203.0.113.5", - src_port=50000, - dst_port=40000, - payload=bytes(request), - ) - - session.process_packet(request_packet(7, 1.0)) - session.process_packet(request_packet(8, 1.1)) - session.process_packet(request_packet(1, 2.0)) - - self.assertEqual([request.page for request in session.pending], [1]) - self.assertEqual( - session.missing_page_reason("permanent", 7), - "request captured; no matching response page was captured", - ) - - timestamp = (2556647947780680000).to_bytes(8, "little") - response = bytearray(0x50) - for _ in range(5): - response += bytes(4) + MARKER + timestamp - self.assertTrue( - session.process_packet( - UdpPacket( - timestamp=2.1, - src_ip="203.0.113.5", - dst_ip="192.168.0.10", - src_port=40000, - dst_port=50000, - payload=bytes(response), - ) - ) - ) - self.assertEqual(session.pairs[-1][0], 1) - self.assertEqual(session.missing_pages("permanent"), []) - - def test_live_session_reports_unrecognized_response_candidate(self): - session = LiveHistorySession("192.168.0.10") - - def request_packet(page, timestamp): - request = bytearray(45) - request[31:35] = (page * 4).to_bytes(4, "little") - request[35:39] = (4220).to_bytes(4, "little") - request[40:44] = (4).to_bytes(4, "little") - return UdpPacket( - timestamp=timestamp, - src_ip="192.168.0.10", - dst_ip="203.0.113.5", - src_port=50000, - dst_port=40000, - payload=bytes(request), - ) - - session.process_packet(request_packet(1, 1.0)) - session.process_packet( - UdpPacket( - timestamp=1.1, - src_ip="203.0.113.5", - dst_ip="192.168.0.10", - src_port=40000, - dst_port=50000, - payload=bytes(220), - ) - ) - session.process_packet(request_packet(2, 1.2)) - - self.assertEqual( - session.missing_page_reason("permanent", 1), - "1 matching inbound UDP packet(s) captured but not recognized as history response (lengths: 220)", - ) - - def test_live_session_ignores_non_history_udp_packets(self): - session = LiveHistorySession("192.168.0.10") - - request = bytearray(45) - request[31:35] = (4).to_bytes(4, "little") - request[35:39] = (4220).to_bytes(4, "little") - request[40:44] = (4).to_bytes(4, "little") - - noise = b"not-a-history-response" * 20 - - session.process_packet( - UdpPacket( - timestamp=1.0, - src_ip="192.168.0.10", - dst_ip="203.0.113.5", - src_port=50000, - dst_port=40000, - payload=bytes(request), - ) - ) - self.assertFalse( - session.process_packet( - UdpPacket( - timestamp=1.1, - src_ip="203.0.113.5", - dst_ip="192.168.0.10", - src_port=40000, - dst_port=50000, - payload=noise, - ) - ) - ) - self.assertEqual(len(session.pairs), 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_capture_backends.py b/tests/test_capture_backends.py new file mode 100644 index 0000000..d768c70 --- /dev/null +++ b/tests/test_capture_backends.py @@ -0,0 +1,92 @@ +from tests.support import * # noqa: F401,F403 +from nte_history_exporter.live_capture.libpcap import _extract_ipv4_frame, _load_library + + +class CaptureBackendTests(unittest.TestCase): + def test_windows_auto_prefers_npcap(self): + capture = Mock(device=r"\Device\NPF_test") + with ( + patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"), + patch( + "nte_history_exporter.live_capture.backends.open_libpcap_capture", + return_value=capture, + ), + patch("nte_history_exporter.live_capture.backends.RawSocketCapture") as raw_capture, + ): + selected = open_capture_backend("192.0.2.1") + + self.assertIs(selected, capture) + self.assertEqual(selected.name, "npcap") + self.assertEqual(selected.fallback_reason, "") + raw_capture.assert_not_called() + + def test_windows_auto_falls_back_to_raw_socket_without_npcap(self): + fallback = Mock() + with ( + patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"), + patch( + "nte_history_exporter.live_capture.backends.open_libpcap_capture", + side_effect=LibpcapUnavailable("Npcap could not be loaded"), + ), + patch( + "nte_history_exporter.live_capture.backends.RawSocketCapture", + return_value=fallback, + ) as raw_capture, + ): + selected = open_capture_backend("192.0.2.1") + + self.assertIs(selected, fallback) + raw_capture.assert_called_once_with( + "192.0.2.1", + fallback_reason="Npcap could not be loaded", + ) + + def test_explicit_libpcap_does_not_fall_back(self): + with ( + patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"), + patch( + "nte_history_exporter.live_capture.backends.open_libpcap_capture", + side_effect=LibpcapUnavailable("Npcap could not be loaded"), + ), + patch("nte_history_exporter.live_capture.backends.RawSocketCapture") as raw_capture, + self.assertRaises(LibpcapUnavailable), + ): + open_capture_backend("192.0.2.1", "libpcap") + + raw_capture.assert_not_called() + + def test_windows_libpcap_load_prefers_system_then_path(self): + system_dir = Path("C:/Windows/System32/Npcap") + loaded = Mock() + with ( + patch("nte_history_exporter.live_capture.libpcap.sys.platform", "win32"), + patch("nte_history_exporter.live_capture.libpcap._windows_npcap_directory", return_value=system_dir), + patch("nte_history_exporter.live_capture.libpcap.Path.is_dir", return_value=True), + patch("nte_history_exporter.live_capture.libpcap.os.add_dll_directory", return_value=Mock(), create=True), + patch( + "nte_history_exporter.live_capture.libpcap.ctypes.CDLL", + side_effect=[OSError("system missing"), loaded], + ) as cdll, + ): + self.assertIs(_load_library(), loaded) + + self.assertEqual( + [call.args[0] for call in cdll.call_args_list], + [ + str(system_dir / "wpcap.dll"), + "wpcap.dll", + ], + ) + + def test_libpcap_link_layers_extract_ipv4_packets(self): + ip_packet = bytes.fromhex("4500001c0000000040110000c0000201c6336402") + bytes(8) + ethernet = bytes(12) + bytes.fromhex("0800") + ip_packet + loop = (2).to_bytes(4, sys.byteorder) + ip_packet + linux_sll = bytes(14) + bytes.fromhex("0800") + ip_packet + linux_sll2 = bytes.fromhex("0800") + bytes(18) + ip_packet + + self.assertEqual(_extract_ipv4_frame(ethernet, DLT_EN10MB), ip_packet) + self.assertEqual(_extract_ipv4_frame(ip_packet, DLT_RAW), ip_packet) + self.assertEqual(_extract_ipv4_frame(loop, DLT_LOOP), ip_packet) + self.assertEqual(_extract_ipv4_frame(linux_sll, DLT_LINUX_SLL), ip_packet) + self.assertEqual(_extract_ipv4_frame(linux_sll2, DLT_LINUX_SLL2), ip_packet) diff --git a/tests/test_export_contract.py b/tests/test_export_contract.py new file mode 100644 index 0000000..24cc919 --- /dev/null +++ b/tests/test_export_contract.py @@ -0,0 +1,108 @@ +from tests.support import * # noqa: F401,F403 +from nte_history_exporter import __version__ + + +class ExportContractTests(unittest.TestCase): + def test_sanitized_export_omits_raw_packet_fields(self): + annotated = annotate_groups(fixture_session().build_rows("permanent")) + export = build_export_json(annotated, []) + + self.assertEqual(export["format"], "nte-history-export") + self.assertIn("exporter", export) + self.assertNotIn("user_uid", export) + self.assertNotIn("record_hex", export["records"][0]) + self.assertNotIn("request_msg", export["records"][0]) + self.assertNotIn("response_msg", export["records"][0]) + + def test_export_includes_user_uid_when_provided(self): + annotated = annotate_groups(fixture_session().build_rows("permanent")) + export = build_export_json( + annotated, + [], + capture_source="npcap", + user_uid="123456789", + ) + + self.assertEqual(list(export).index("user_uid"), list(export).index("records") - 1) + self.assertEqual(export["capture_source"], "npcap") + self.assertEqual(export["user_uid"], "123456789") + + def test_debug_csv_includes_exporter_version(self): + with TemporaryDirectory() as tmp: + path = Path(tmp) / "debug.csv" + write_csv(path, [{"uid": "abc123"}]) + + with path.open(newline="", encoding="utf-8") as f: + rows = list(csv.DictReader(f)) + + self.assertEqual(rows[0]["exporter_version"], __version__) + self.assertEqual(rows[0]["uid"], "abc123") + + def test_export_paths_include_user_uid_banner_and_timestamp(self): + _csv_path, json_path = export_paths("limited_character", "218216016349") + + self.assertRegex( + json_path.name, + r"^218216016349_Limited_\d{8}_\d{6}(?:_\d+)?\.json$", + ) + + def test_extracts_user_uid_from_record_context(self): + payload = ( + b"\x00" * 24 + + (218216016349).to_bytes(8, "little") + + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00" + ) + + self.assertEqual(extract_user_uid(payload), "218216016349") + + def test_extracts_user_uid_from_private_spawn_record_context(self): + payload = ( + b"\x88\x00\x00\x00\x10\x00\x00\x00" + + (218216016349).to_bytes(8, "little") + + b"\x08\x00\x0c\x00\x07\x00\x08\x00\x08\x00\x00\x00" + + b"\x00\x00\x00\x01\x08\x00\x00\x00\x04\x00\x04\x00" + + b"\x04\x00\x00\x00\x16\x00\x00\x00PrivateSpawnInfoRecord\x00" + ) + + self.assertEqual(extract_user_uid(payload), "218216016349") + + def test_does_not_extract_user_uid_from_wrong_record_offset(self): + payload = ( + b"\x00" * 28 + + (218216016349).to_bytes(8, "little") + + b"\x00\x00\x00\x00TagOthers\x00" + ) + + self.assertIsNone(extract_user_uid(payload)) + + def test_does_not_extract_old_eight_digit_false_positive_as_user_uid(self): + payload = ( + b"WholeVehicleData\x00\x00\x00\x00\x00o<\x00\x00\x05\x00\x00\x00" + b"\x0b\x00\x00\x00Vehicle015\x00\x0b\x00\x00\x00buyvehicle\x00" + b"\x09\x00\x00\x0015363624\x00\x06\x00\x00\x00" + ) + + self.assertIsNone(extract_user_uid(payload)) + + def test_ipv4_parser_extracts_tcp_payload_for_user_uid_detection(self): + payload = ( + (218216016349).to_bytes(8, "little") + + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00" + ) + tcp_header = bytearray(20) + tcp_header[0:2] = (40000).to_bytes(2, "big") + tcp_header[2:4] = (30000).to_bytes(2, "big") + tcp_header[12] = 5 << 4 + total_len = 20 + len(tcp_header) + len(payload) + ip_header = bytearray(20) + ip_header[0] = 0x45 + ip_header[2:4] = total_len.to_bytes(2, "big") + ip_header[9] = 6 + ip_header[12:16] = bytes([192, 0, 2, 1]) + ip_header[16:20] = bytes([198, 51, 100, 2]) + + packet = parse_ipv4_packet(bytes(ip_header) + bytes(tcp_header) + payload) + + self.assertIsNotNone(packet) + self.assertEqual(packet.protocol, "tcp") + self.assertEqual(packet.payload, payload) diff --git a/tests/test_live_session.py b/tests/test_live_session.py new file mode 100644 index 0000000..b92cc92 --- /dev/null +++ b/tests/test_live_session.py @@ -0,0 +1,214 @@ +from tests.support import * # noqa: F401,F403 + + +class LiveSessionTests(unittest.TestCase): + def test_live_session_pairs_request_and_response(self): + session = LiveHistorySession("192.168.0.10") + + request = bytearray(45) + request[31:35] = (4).to_bytes(4, "little") + request[35:39] = (4220).to_bytes(4, "little") + request[40:44] = (4).to_bytes(4, "little") + + response = bytearray(220) + response[0x50:0x50 + len(MARKER)] = MARKER + response[0x50 + len(MARKER):0x50 + len(MARKER) + 8] = ( + 2556647947780680000 + ).to_bytes(8, "little") + + self.assertFalse( + session.process_packet( + UdpPacket( + timestamp=1.0, + src_ip="192.168.0.10", + dst_ip="203.0.113.5", + src_port=50000, + dst_port=40000, + payload=bytes(request), + ) + ) + ) + self.assertTrue( + session.process_packet( + UdpPacket( + timestamp=1.2, + src_ip="203.0.113.5", + dst_ip="192.168.0.10", + src_port=40000, + dst_port=50000, + payload=bytes(response), + ) + ) + ) + self.assertEqual(len(session.pairs), 1) + self.assertEqual(session.last_page_seen, 1) + + def test_live_session_pairs_pipelined_and_batched_pages(self): + session = LiveHistorySession("192.168.0.10") + + def request_packet(page, timestamp): + request = bytearray(45) + request[31:35] = (page * 4).to_bytes(4, "little") + request[35:39] = (4220).to_bytes(4, "little") + request[40:44] = (4).to_bytes(4, "little") + return UdpPacket( + timestamp=timestamp, + src_ip="192.168.0.10", + dst_ip="203.0.113.5", + src_port=50000, + dst_port=40000, + payload=bytes(request), + ) + + timestamp = (2556647947780680000).to_bytes(8, "little") + + def response_packet(record_count, packet_timestamp): + response = bytearray(0x50) + for _ in range(record_count): + response += bytes(4) + MARKER + timestamp + return UdpPacket( + timestamp=packet_timestamp, + src_ip="203.0.113.5", + dst_ip="192.168.0.10", + src_port=40000, + dst_port=50000, + payload=bytes(response), + ) + + for page in range(1, 9): + session.process_packet(request_packet(page, 1.0 + page / 10)) + + self.assertTrue(session.process_packet(response_packet(10, 2.0))) + self.assertEqual([pair[0] for pair in session.pairs], [1, 2]) + self.assertEqual([pair[8:10] for pair in session.pairs], [(0, 5), (5, 5)]) + + self.assertTrue(session.process_packet(response_packet(10, 2.1))) + self.assertEqual([pair[0] for pair in session.pairs], [1, 2, 3, 4]) + self.assertEqual(session.missing_pages("permanent"), []) + rows = session.build_rows("permanent") + self.assertEqual(len(rows), 20) + self.assertEqual( + {page: sum(row["page"] == page for row in rows) for page in range(1, 5)}, + {1: 5, 2: 5, 3: 5, 4: 5}, + ) + + session.process_packet(request_packet(9, 2.2)) + self.assertTrue(session.process_packet(response_packet(4, 2.3))) + self.assertEqual(session.pairs[-1][0], 9) + self.assertEqual(session.pairs[-1][8:10], (0, 4)) + + def test_live_session_new_page_one_starts_clean_recovery_cycle(self): + session = LiveHistorySession("192.168.0.10") + + def request_packet(page, timestamp): + request = bytearray(45) + request[31:35] = (page * 4).to_bytes(4, "little") + request[35:39] = (4220).to_bytes(4, "little") + request[40:44] = (4).to_bytes(4, "little") + return UdpPacket( + timestamp=timestamp, + src_ip="192.168.0.10", + dst_ip="203.0.113.5", + src_port=50000, + dst_port=40000, + payload=bytes(request), + ) + + session.process_packet(request_packet(7, 1.0)) + session.process_packet(request_packet(8, 1.1)) + session.process_packet(request_packet(1, 2.0)) + + self.assertEqual([request.page for request in session.pending], [1]) + self.assertEqual( + session.missing_page_reason("permanent", 7), + "request captured; no matching response page was captured", + ) + + timestamp = (2556647947780680000).to_bytes(8, "little") + response = bytearray(0x50) + for _ in range(5): + response += bytes(4) + MARKER + timestamp + self.assertTrue( + session.process_packet( + UdpPacket( + timestamp=2.1, + src_ip="203.0.113.5", + dst_ip="192.168.0.10", + src_port=40000, + dst_port=50000, + payload=bytes(response), + ) + ) + ) + self.assertEqual(session.pairs[-1][0], 1) + self.assertEqual(session.missing_pages("permanent"), []) + + def test_live_session_reports_unrecognized_response_candidate(self): + session = LiveHistorySession("192.168.0.10") + + def request_packet(page, timestamp): + request = bytearray(45) + request[31:35] = (page * 4).to_bytes(4, "little") + request[35:39] = (4220).to_bytes(4, "little") + request[40:44] = (4).to_bytes(4, "little") + return UdpPacket( + timestamp=timestamp, + src_ip="192.168.0.10", + dst_ip="203.0.113.5", + src_port=50000, + dst_port=40000, + payload=bytes(request), + ) + + session.process_packet(request_packet(1, 1.0)) + session.process_packet( + UdpPacket( + timestamp=1.1, + src_ip="203.0.113.5", + dst_ip="192.168.0.10", + src_port=40000, + dst_port=50000, + payload=bytes(220), + ) + ) + session.process_packet(request_packet(2, 1.2)) + + self.assertEqual( + session.missing_page_reason("permanent", 1), + "1 matching inbound UDP packet(s) captured but not recognized as history response (lengths: 220)", + ) + + def test_live_session_ignores_non_history_udp_packets(self): + session = LiveHistorySession("192.168.0.10") + + request = bytearray(45) + request[31:35] = (4).to_bytes(4, "little") + request[35:39] = (4220).to_bytes(4, "little") + request[40:44] = (4).to_bytes(4, "little") + + noise = b"not-a-history-response" * 20 + + session.process_packet( + UdpPacket( + timestamp=1.0, + src_ip="192.168.0.10", + dst_ip="203.0.113.5", + src_port=50000, + dst_port=40000, + payload=bytes(request), + ) + ) + self.assertFalse( + session.process_packet( + UdpPacket( + timestamp=1.1, + src_ip="203.0.113.5", + dst_ip="192.168.0.10", + src_port=40000, + dst_port=50000, + payload=noise, + ) + ) + ) + self.assertEqual(len(session.pairs), 0) + diff --git a/tests/test_mappings_and_updates.py b/tests/test_mappings_and_updates.py new file mode 100644 index 0000000..4b9cb35 --- /dev/null +++ b/tests/test_mappings_and_updates.py @@ -0,0 +1,122 @@ +from tests.support import * # noqa: F401,F403 + + +class MappingAndUpdateTests(unittest.TestCase): + def test_pool_mapping_json_files_have_uniform_shape(self): + required_top_level = {"pool_key", "game", "system", "banner", "request", "response"} + for pool_key, mapping in load_pool_mappings().items(): + with self.subTest(pool_key=pool_key): + self.assertEqual(set(required_top_level) - set(mapping), set()) + self.assertEqual(mapping["pool_key"], pool_key) + self.assertIn("id", mapping["system"]) + self.assertIn("name", mapping["system"]) + self.assertIn("id", mapping["banner"]) + self.assertIn("name", mapping["banner"]) + self.assertIn("shared_pity", mapping["banner"]) + self.assertIn("family", mapping["request"]) + self.assertIn("length", mapping["request"]) + self.assertIn("constant", mapping["request"]) + self.assertIn("cursor_step", mapping["request"]) + + def test_pool_mapping_json_matches_runtime_pool_meta(self): + for pool_key, mapping in load_pool_mappings().items(): + with self.subTest(pool_key=pool_key): + self.assertEqual(pool_meta_from_mapping(mapping), POOL_META[pool_key]) + + def test_update_version_comparison_handles_release_tags(self): + self.assertTrue(is_newer_version("v0.1.7", "0.1.6")) + self.assertTrue(is_newer_version("0.2.0", "0.1.6")) + self.assertTrue(is_newer_version("v0.1.10", "0.1.9")) + self.assertFalse(is_newer_version("v0.1.6", "0.1.6")) + self.assertFalse(is_newer_version("v0.1.5", "0.1.6")) + self.assertFalse(is_newer_version("latest", "0.1.6")) + + def test_update_check_reports_newer_github_release(self): + latest = { + "tag_name": "v0.1.7", + "html_url": "https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.7", + } + with patch("nte_history_exporter.update_check.fetch_latest_release", return_value=latest): + update = check_for_update("0.1.6", timeout=0.1) + + self.assertEqual( + update, + UpdateInfo( + current_version="0.1.6", + latest_version="v0.1.7", + release_url="https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.7", + ), + ) + + def test_update_check_ignores_prerelease(self): + latest = { + "tag_name": "v0.1.8-dev-branch.123", + "html_url": "https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.8-dev-branch.123", + "prerelease": True, + } + with patch("nte_history_exporter.update_check.fetch_latest_release", return_value=latest): + self.assertIsNone(check_for_update("0.1.7", timeout=0.1)) + + def test_update_check_is_quiet_when_unavailable_or_current(self): + with patch("nte_history_exporter.update_check.fetch_latest_release", side_effect=OSError("offline")): + self.assertIsNone(check_for_update("0.1.6", timeout=0.1)) + + with patch("nte_history_exporter.update_check.fetch_latest_release", return_value={"tag_name": "v0.1.6"}): + self.assertIsNone(check_for_update("0.1.6", timeout=0.1)) + + def test_reward_mapping_files_have_expected_shape(self): + self.assertTrue(ARC_META) + for arc_id, meta in ARC_META.items(): + with self.subTest(arc_id=arc_id): + self.assertTrue(arc_id.startswith("fork_")) + self.assertIn("name", meta) + self.assertIn(meta.get("rank"), ("S", "A", "B")) + + self.assertTrue(CHARACTERS) + for character_id, info in CHARACTERS.items(): + with self.subTest(character_id=character_id): + self.assertTrue(character_id.isdigit()) + self.assertIn("name", info) + self.assertIn(info.get("rank"), ("S", "A")) + + self.assertTrue(ITEMS) + for item_id, info in ITEMS.items(): + with self.subTest(item_id=item_id): + self.assertIn(info.get("type"), ("item", "cosmetic")) + self.assertIn("name", info) + + def test_rewards_by_id_merges_all_mapping_files(self): + for reward_id in (*ARC_META, *CHARACTERS, *ITEMS): + with self.subTest(reward_id=reward_id): + reward = REWARDS_BY_ID[reward_id] + self.assertEqual(reward["id"], reward_id) + self.assertIn(reward["type"], ("arc", "character", "item", "cosmetic")) + + def test_decode_reward_key_round_trips_observed_keys(self): + observed = { + "98bdc9ad7dd9a5b99501": "fork_vine", + "98bdc9ad7d41c9bdad85c9e5bdb901": "fork_Prokaryon", + "98bdc9ad7ddda1d585add585b99d01": "fork_whuakuang", + "98bdc9ad7dddd5a1d585add585b99d01": "fork_wuhuakuang", + "10a58d9539bdc9b585b101": "DiceNormal", + "10a58d957dd1a58dad95d17dc1c400": "Dice_ticket_01", + "10a58d957dd1a58dad95d17dc1c800": "Dice_ticket_02", + "10a58d95b1a5b5a5d19501": "Dicelimite", + "1885cda1a5bdb97d1db1a591957dc5c0c4c000": "Fashion_Glide_1010", + "1885cda1a5bdb97dd995a1a58db1957dc5c0c4c07c59c1c0e000": "Fashion_vehicle_1010_V008", + "c4c0cccc00": "1033", + "c4c0dcc000": "1070", + "c4c0dcc0": "1070", + "c4c0c8c4": "1021", + } + for key_hex, expected_id in observed.items(): + with self.subTest(key_hex=key_hex): + self.assertEqual(decode_reward_key(bytes.fromhex(key_hex)), expected_id) + + def test_infer_reward_type_for_unmapped_ids(self): + self.assertEqual(infer_reward_type("fork_newarc"), "arc") + self.assertEqual(infer_reward_type("1099"), "character") + self.assertEqual(infer_reward_type("Fashion_hat_2000"), "cosmetic") + self.assertEqual(infer_reward_type("Dice_ticket_03"), "item") + self.assertEqual(infer_reward_type(""), "") + diff --git a/tests/test_network_fixture.py b/tests/test_network_fixture.py new file mode 100644 index 0000000..1a4be10 --- /dev/null +++ b/tests/test_network_fixture.py @@ -0,0 +1,74 @@ +from tests.support import * # noqa: F401,F403 + + +class SyntheticNetworkFixtureTests(unittest.TestCase): + def test_fixture_contains_only_synthetic_network_identity(self): + fixture = load_network_fixture() + + self.assertTrue(fixture["privacy"]["synthetic"]) + self.assertFalse(fixture["privacy"]["contains_user_uid"]) + self.assertFalse(fixture["privacy"]["contains_raw_account_session"]) + self.assertEqual(fixture["local_ip"], "192.0.2.10") + + allowed_ips = {"192.0.2.10", "198.51.100.20"} + for packet, decoded in zip(fixture["packets"], fixture_packets()): + with self.subTest(label=packet["label"]): + self.assertIn(packet["src_ip"], allowed_ips) + self.assertIn(packet["dst_ip"], allowed_ips) + self.assertEqual(packet["protocol"], "udp") + self.assertIsNone(extract_user_uid(decoded.payload)) + + def test_fixture_replays_permanent_and_arc_history(self): + fixture = load_network_fixture() + session = fixture_session() + + permanent = annotate_groups(session.build_rows("permanent")) + arc = session.build_rows("arc_miracle_box") + + self.assertIsNone(session.user_uid) + self.assertEqual(len(permanent), fixture["expected"]["permanent_records"]) + self.assertEqual(len(arc), fixture["expected"]["arc_records"]) + self.assertEqual(permanent[0]["uid"], fixture["expected"]["permanent_first_uid"]) + self.assertEqual(arc[0]["uid"], fixture["expected"]["arc_first_uid"]) + self.assertEqual(sorted({row["page"] for row in permanent}), [1, 2, 3, 4, 5]) + self.assertEqual(sorted({row["page"] for row in arc}), [1, 2, 3, 4, 5]) + self.assertEqual({row["timestamp_decoded"] for row in permanent}, {"2030-01-01 00:00:00"}) + self.assertEqual({row["timestamp_decoded"] for row in arc}, {"2030-01-02 00:00:00"}) + + # Repeating one manufactured record prevents the replay transcript from + # preserving a real account's pull sequence while still exercising page + # ordering, timestamp ordinals, UID stability, and export behavior. + self.assertEqual(len({row["record_hex"] for row in permanent}), 1) + self.assertEqual(len({row["record_hex"] for row in arc}), 1) + + def test_fixture_protocol_samples_cover_edge_cases(self): + points = decode_response_records(fixture_payload("limited-points-gift-1")) + chase = decode_response_records(fixture_payload("limited-chase-reward")) + batched = decode_response_records(fixture_payload("limited-batched-pages")) + aligned = decode_response_records(fixture_payload("limited-bitpacked-source")) + bitpacked = decode_response_records(fixture_payload("limited-bitpacked-response")) + + self.assertEqual(len(points), 1) + self.assertEqual(points[0]["result_type"], "points_gift") + self.assertEqual(points[0]["reward_id"], "1020") + self.assertEqual(len(chase), 1) + self.assertEqual(chase[0]["result_type"], "chase_reward") + self.assertEqual(chase[0]["reward_id"], "Dice_ticket_01") + self.assertEqual(chase[0]["quantity"], 30) + self.assertEqual(len(batched), 10) + self.assertEqual(len(aligned), 10) + self.assertEqual( + [row["record_hex"] for row in bitpacked], + [row["record_hex"] for row in aligned], + ) + + def test_fixture_exports_sanitized_json_with_stable_uids(self): + session = fixture_session() + rows = annotate_groups(session.build_rows("permanent")) + export = build_export_json(rows, []) + + self.assertNotIn("user_uid", export) + self.assertEqual(export["format_version"], 1) + self.assertEqual(export["records"][0]["uid"], load_network_fixture()["expected"]["permanent_first_uid"]) + self.assertNotIn("record_hex", export["records"][0]) + self.assertNotIn("timestamp_raw_hex", export["records"][0]) diff --git a/tests/test_protocol_decoding.py b/tests/test_protocol_decoding.py new file mode 100644 index 0000000..22b1840 --- /dev/null +++ b/tests/test_protocol_decoding.py @@ -0,0 +1,125 @@ +from tests.support import * # noqa: F401,F403 + + +class ProtocolDecodingTests(unittest.TestCase): + def test_limited_selector_and_marker_decode(self): + request = bytearray(45) + request[31:35] = (4).to_bytes(4, "little") + request[35:39] = (4220).to_bytes(4, "little") + request[40:44] = (8).to_bytes(4, "little") + self.assertEqual(history_request_kind(bytes(request)), "limited_character") + + def test_monopoly_request_allows_coalesced_trailing_payload(self): + request = bytearray(45) + request[31:35] = (25 * 4).to_bytes(4, "little") + request[35:39] = (4220).to_bytes(4, "little") + request[40:44] = (4).to_bytes(4, "little") + coalesced = bytes(request) + bytes.fromhex( + "007c669610062038461bc40100000872a34b93821a0219aa933b0a6b2ba34a6b" + ) + + self.assertEqual(history_request_kind(coalesced), "permanent") + + def test_arc_request_allows_coalesced_trailing_payload(self): + request = bytearray(34) + request[24:28] = (2060).to_bytes(4, "little") + request[29:33] = (7 * 2).to_bytes(4, "little") + + self.assertTrue(is_arc_history_request(bytes(request) + bytes(32))) + self.assertEqual(arc_request_page(bytes(request) + bytes(32)), 7) + + response = bytearray(220) + response[0x50:0x54] = (4).to_bytes(4, "little") + response[0x54:0x58] = (20).to_bytes(4, "little") + response[0x58:0x5d] = bytes.fromhex("c4c0c4c000") + marker_offset = 0x5d + response[marker_offset:marker_offset + len(LIMITED_CHARACTER_MARKER)] = LIMITED_CHARACTER_MARKER + timestamp_raw = (2556647947780680000).to_bytes(8, "little") + response[marker_offset + len(LIMITED_CHARACTER_MARKER):marker_offset + len(LIMITED_CHARACTER_MARKER) + 8] = timestamp_raw + + rows = decode_response_records(bytes(response)) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["reward_id"], "1010") + self.assertEqual(rows[0]["reward_name"], "Nanally") + + def test_fixture_prefixed_points_gift_overrides_visible_dice(self): + decoded = decode_response_records(fixture_payload("limited-points-gift-1"))[0] + self.assertEqual(decoded["result_type"], "points_gift") + self.assertEqual(decoded["result_source_raw"], 0) + self.assertEqual(decoded["dice"], 0) + self.assertEqual(decoded["dice_raw_u32"], 0) + self.assertEqual(decoded["reward_id"], "1020") + + def test_fixture_prefixed_chase_reward_overrides_visible_dice_and_quantity(self): + decoded = decode_response_records(fixture_payload("limited-chase-reward"))[0] + + self.assertEqual(decoded["result_type"], "chase_reward") + self.assertEqual(decoded["result_source_raw"], -4) + self.assertEqual(decoded["dice"], -4) + self.assertEqual(decoded["dice_raw_u32"], -4) + self.assertEqual(decoded["reward_id"], "Dice_ticket_01") + self.assertEqual(decoded["quantity"], 30) + + def test_warp_piece_chase_subrecord_without_prefix_marker_is_chase_reward(self): + decoded = decode_single_record( + "c1c4b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c400" + "4c0000000c85c99141bdbdb17d0da185c9858dd195c90140eb2c2dd7227b23" + ) + + self.assertEqual(decoded["result_type"], "chase_reward") + self.assertEqual(decoded["result_source_raw"], -4) + self.assertEqual(decoded["dice"], -4) + self.assertEqual(decoded["dice_raw_u32"], -4) + self.assertEqual(decoded["reward_id"], "Dice_ticket_01") + self.assertEqual(decoded["reward_name"], "Warp Piece") + self.assertEqual(decoded["quantity"], 30) + + def test_page_first_prefix_uses_real_dice_field(self): + cases = [ + ( + "003006000014000000040000002800000098bdc9ad7dd9a5b995010000000008000000" + "3c00000010a58d957dd1a58dad95d17dc1c4002800000098bdc9ad7dd9a5b995014c" + "0000000c85c99141bdbdb17d0da185c9858dd195c901c0dd53bd2b137b23", + 1, + "fork_vine", + ), + ( + "00c8060000140000001000000014000000c4c0d4d400000000000400000014000000" + "c4c0d4d400440000000c85c99141bdbdb17d3995dd49bdb1950100d929e115087b23", + 4, + "1055", + ), + ] + for record_hex, expected_dice, reward_id in cases: + with self.subTest(reward_id=reward_id): + decoded = decode_single_record(record_hex) + + self.assertEqual(decoded["dice"], expected_dice) + self.assertEqual(decoded["dice_raw_u32"], expected_dice * 4) + self.assertEqual(decoded["dice_offset_in_record"], 9) + self.assertEqual(decoded["result_type"], "dice") + self.assertEqual(decoded["reward_id"], reward_id) + + def test_batched_monopoly_response_normalizes_embedded_page_header(self): + decoded = decode_response_records(fixture_payload("limited-batched-pages")) + + self.assertEqual(len(decoded), 10) + self.assertEqual(decoded[5]["dice"], 4) + self.assertEqual(decoded[5]["result_type"], "dice") + + def test_monopoly_response_parser_realigns_bit_packed_payload(self): + reference_rows = decode_response_records(fixture_payload("limited-bitpacked-source")) + decoded = decode_response_records(fixture_payload("limited-bitpacked-response")) + + self.assertEqual(len(decoded), 10) + self.assertEqual( + [row["record_hex"] for row in decoded], + [row["record_hex"] for row in reference_rows], + ) + + def test_page_gap_warning_reports_ignored_pages(self): + pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (1, 2, 3, 5)] + run, warnings = select_continuous_run_from_page_1(pairs) + self.assertEqual([p[0] for p in run], [1, 2, 3]) + self.assertEqual(warnings[0]["code"], "PAGE_GAP_DETECTED") + self.assertEqual(warnings[0]["ignored_pages"], [5]) diff --git a/tests/test_uid_compatibility.py b/tests/test_uid_compatibility.py new file mode 100644 index 0000000..ed7d3af --- /dev/null +++ b/tests/test_uid_compatibility.py @@ -0,0 +1,153 @@ +from tests.support import * # noqa: F401,F403 + + +class UidCompatibilityTests(unittest.TestCase): + def test_uid_source_matches_committed_network_fixture(self): + fixture = load_network_fixture() + rows = annotate_groups(fixture_session().build_rows("permanent")) + self.assertEqual(rows[0]["uid"], fixture["expected"]["permanent_first_uid"]) + + def test_uid_uses_pool_timestamp_and_ordinal_only(self): + row = { + "pool_group_id": "Lottery_LimitedCharacter", + "timestamp_raw_hex": "40e93247c3097b23", + "dice": 5, + "reward_key_hex": "10a58d957dd1a58dad95d17dc1c800", + "quantity": 50, + } + changed_content = { + **row, + "dice": 1, + "reward_key_hex": "98bdc9ad7dd9a5b99501", + "quantity": 1, + } + changed_pool = {**row, "pool_group_id": "Lottery_Permanent"} + + self.assertEqual(make_uid(row, 0), "74a9ef4aacde549dfe8e8e7cc6ddd65b") + self.assertEqual(make_uid(changed_content, 0), make_uid(row, 0)) + self.assertNotEqual(make_uid(changed_pool, 0), make_uid(row, 0)) + self.assertNotEqual(make_uid(row, 1), make_uid(row, 0)) + + def test_pages_1_to_5_exports_every_row(self): + annotated = annotate_groups(fixture_session().build_rows("permanent")) + + exported = [row for row in annotated if row["export_record"] is True] + + # Every decoded row is exported; boundary groups are never dropped. + self.assertEqual(len(annotated), 25) + self.assertEqual(len(exported), 25) + + @staticmethod + def _synthetic_row(page, timestamp_hex, result_type): + return { + "page": page, + "timestamp_raw_hex": timestamp_hex, + "timestamp_decoded": f"ts-{timestamp_hex}", + "result_type": result_type, + "dice": 4 if result_type == "dice" else 0, + "reward_key_hex": "10a58d9539bdc9b585b101", + "quantity": 1, + } + + def test_oldest_group_with_partial_dice_count_exports_without_warning(self): + rows = [self._synthetic_row(1, "aa", "dice") for _ in range(5)] + rows += [self._synthetic_row(2, "bb", "dice") for _ in range(5)] + rows += [self._synthetic_row(3, "bb", "dice") for _ in range(4)] + rows += [self._synthetic_row(3, "bb", "points_gift")] + + annotated = annotate_groups(rows) + exported = [row for row in annotated if row["export_record"] is True] + + # Oldest group is a partially captured 10-pull on a full final page. Its + # captured prefix is ordinal-stable, so it is exported with stable UIDs. + self.assertEqual(len(exported), 15) + oldest = [row for row in annotated if row["timestamp_raw_hex"] == "bb"] + self.assertTrue(all(row["uid"] for row in oldest)) + self.assertTrue(all(row["uid_status"] == "stable" for row in oldest)) + self.assertEqual([row["timestamp_group_ordinal"] for row in oldest], list(range(10))) + + def test_incomplete_oldest_prefix_keeps_stable_uids(self): + full = [self._synthetic_row(1, "aa", "dice") for _ in range(5)] + full += [self._synthetic_row(2, "bb", "dice") for _ in range(3)] + full += [self._synthetic_row(3, "bb", "dice") for _ in range(2)] + truncated = [r for r in full if r["page"] in (1, 2)] + + full_rows = annotate_groups([dict(r) for r in full]) + trunc_rows = annotate_groups([dict(r) for r in truncated]) + + full_uids = [r["uid"] for r in full_rows if r["timestamp_raw_hex"] == "bb"][:3] + trunc_uids = [r["uid"] for r in trunc_rows if r["timestamp_raw_hex"] == "bb"] + # Capturing only the first 3 of a 5-record oldest group yields the same + # UIDs those rows have in the full capture. + self.assertEqual(len(trunc_uids), 3) + self.assertEqual(trunc_uids, full_uids) + + def test_oldest_group_with_ten_dice_exports_on_full_final_page(self): + rows = [self._synthetic_row(1, "aa", "dice") for _ in range(5)] + rows += [self._synthetic_row(2, "bb", "dice") for _ in range(5)] + rows += [self._synthetic_row(3, "bb", "dice") for _ in range(5)] + + annotated = annotate_groups(rows) + exported = [row for row in annotated if row["export_record"] is True] + + self.assertEqual(len(exported), 15) + + def test_run_selection_anchors_to_page_1_and_keeps_newest(self): + # Page 2's response was lost: captured pages 1, 3, 4, 5. + pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (1, 3, 4, 5)] + run, warnings = select_continuous_run_from_page_1(pairs) + + # The page-1 run (just page 1, the newest history) is kept; later pages are + # ignored with a gap warning, never silently discarding page 1. + self.assertEqual([p[0] for p in run], [1]) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0]["code"], "PAGE_GAP_DETECTED") + self.assertEqual(warnings[0]["ignored_pages"], [3, 4, 5]) + + def test_run_selection_warns_when_page_1_missing(self): + pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (3, 4, 5)] + run, warnings = select_continuous_run_from_page_1(pairs) + + self.assertEqual([p[0] for p in run], [3, 4, 5]) + self.assertEqual(warnings[0]["code"], "DID_NOT_START_AT_PAGE_1") + + def test_committed_network_scan_exports_all_rows(self): + fixture = load_network_fixture() + annotated = annotate_groups(fixture_session().build_rows("permanent")) + exported = [row for row in annotated if row["export_record"] is True] + + self.assertEqual(len(annotated), fixture["expected"]["permanent_records"]) + self.assertEqual(len(exported), fixture["expected"]["permanent_records"]) + + def test_monopoly_uid_compatibility_vectors_are_frozen(self): + vectors = [ + ("Lottery_Permanent", "0000000000000000", 0, "5da57468adb23fcfde530fd849a25767"), + ("Lottery_Permanent", "00a243eb689b7a23", 0, "8f9fb9bc92a867f6f41e5839e94c7991"), + ("Lottery_Permanent", "00a243eb689b7a23", 1, "6a0bea71d091b9b7063780fcc86238d5"), + ("Lottery_LimitedCharacter", "00a243eb689b7a23", 0, "b8f5d3c27315c70ceafbe8eb2bab0fc2"), + ("Lottery_LimitedCharacter", "ffffffffffffffff", 9, "66916e1f077fdf7664f0f0edc4c64eb8"), + ] + + for pool_group_id, timestamp_raw_hex, ordinal, expected_uid in vectors: + with self.subTest( + pool_group_id=pool_group_id, + timestamp=timestamp_raw_hex, + ordinal=ordinal, + ): + row = { + "pool_group_id": pool_group_id, + "timestamp_raw_hex": timestamp_raw_hex, + } + self.assertEqual(make_uid(row, ordinal), expected_uid) + + def test_arc_uid_compatibility_vectors_are_frozen(self): + vectors = [ + ("0000000000000000", 0, "faa0b07cefd71758d4a8d64bc2a2f78e"), + ("00a243eb689b7a23", 0, "7a7ef75281fcced043f691d269477cfa"), + ("00a243eb689b7a23", 1, "f798c076ea1d137a08fe62df859e8a7f"), + ("ffffffffffffffff", 9, "d114cc6c313a15f55ca12910801d78ea"), + ] + + for timestamp_raw_hex, ordinal, expected_uid in vectors: + with self.subTest(timestamp=timestamp_raw_hex, ordinal=ordinal): + self.assertEqual(make_arc_uid(timestamp_raw_hex, ordinal), expected_uid) From 042d47e3d44d8471f74a62eb8a10578ee50b9e2d Mon Sep 17 00:00:00 2001 From: Golumpa Date: Wed, 15 Jul 2026 19:06:47 +0100 Subject: [PATCH 2/8] Added mappings updater Implemented a mappings updater script that uses the Waifus-Grace/NTE_Assets repo to export all the required mappings. --- .github/workflows/update-mappings.yml | 136 ++++++ README.md | 20 + docs/mapping-updates.md | 95 +++++ mappings/arcs.json | 36 +- mappings/characters.json | 1 + mappings/items.json | 41 +- src/nte_history_exporter/mapping_update.py | 387 ++++++++++++++++++ tests/fixtures/README.md | 9 + .../Appearance/DT_AppearanceData.json | 1 + .../DataTable/Character/DT_Character.json | 1 + .../DataTable/Fork/DT_ForkItemData.json | 1 + .../DataTable/Gacha/GachaIllustrate.json | 1 + .../Inventory/DT_CapitalItemConfig.json | 1 + .../DataTable/Inventory/DT_ItemConfig.json | 1 + .../nte_assets/Localization/en/game.json | 18 + tests/test_mapping_update.py | 184 +++++++++ tools/update_mappings.py | 86 ++++ 17 files changed, 980 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/update-mappings.yml create mode 100644 docs/mapping-updates.md create mode 100644 src/nte_history_exporter/mapping_update.py create mode 100644 tests/fixtures/nte_assets/DataTable/Character/Appearance/DT_AppearanceData.json create mode 100644 tests/fixtures/nte_assets/DataTable/Character/DT_Character.json create mode 100644 tests/fixtures/nte_assets/DataTable/Fork/DT_ForkItemData.json create mode 100644 tests/fixtures/nte_assets/DataTable/Gacha/GachaIllustrate.json create mode 100644 tests/fixtures/nte_assets/DataTable/Inventory/DT_CapitalItemConfig.json create mode 100644 tests/fixtures/nte_assets/DataTable/Inventory/DT_ItemConfig.json create mode 100644 tests/fixtures/nte_assets/Localization/en/game.json create mode 100644 tests/test_mapping_update.py create mode 100644 tools/update_mappings.py diff --git a/.github/workflows/update-mappings.yml b/.github/workflows/update-mappings.yml new file mode 100644 index 0000000..8760bb3 --- /dev/null +++ b/.github/workflows/update-mappings.yml @@ -0,0 +1,136 @@ +name: Update reward mappings + +on: + workflow_dispatch: + inputs: + source_ref: + description: NTE_Assets branch, tag, or commit + required: true + default: main + type: string + +permissions: + contents: write + pull-requests: write + +concurrency: + group: update-reward-mappings + cancel-in-progress: false + +jobs: + update: + name: Rebuild mappings from NTE_Assets + runs-on: ubuntu-latest + + env: + AUTOMATION_BRANCH: dev/automated-mapping-update + NTE_ASSETS_REF: ${{ inputs.source_ref }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout source code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + ref: ${{ github.event.repository.default_branch }} + + - name: Get Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install project + run: python -m pip install -e . + + - name: Rebuild reward mappings + run: python tools/update_mappings.py --source-ref "$NTE_ASSETS_REF" --apply + + - name: Run tests + run: python -m unittest discover -s tests + + - name: Summarize mapping changes + id: changes + shell: bash + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + report = json.loads(Path("build/mapping-update/mapping-update-report.json").read_text()) + changes = report["changes"] + changed = sum(changes.values()) > 0 + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output: + output.write(f"changed={'true' if changed else 'false'}\n") + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as summary: + summary.write("## NTE_Assets mapping update\n\n") + summary.write(f"Source ref: `{report['source_ref']}`\n\n") + summary.write(f"- Additions: {changes['additions']}\n") + summary.write(f"- Updates: {changes['updates']}\n") + summary.write(f"- Deletions: {changes['deletions']}\n") + PY + + - name: Upload review report + uses: actions/upload-artifact@v6 + with: + name: mapping-update-report + path: build/mapping-update/mapping-update-report.json + + - name: Commit and push mapping changes + if: steps.changes.outputs.changed == 'true' + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$AUTOMATION_BRANCH" + git add mappings/arcs.json mappings/characters.json mappings/items.json + git commit -m "chore: update reward mappings from NTE_Assets" + git push --force origin "HEAD:refs/heads/$AUTOMATION_BRANCH" + + - name: Open or update pull request + if: steps.changes.outputs.changed == 'true' + shell: bash + env: + BASE_BRANCH: ${{ github.event.repository.default_branch }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + report = json.loads(Path("build/mapping-update/mapping-update-report.json").read_text()) + changes = report["changes"] + body = ( + "Automated authoritative reward-mapping snapshot from " + "[Waifus-Grace/NTE_Assets](https://github.com/Waifus-Grace/NTE_Assets).\n\n" + f"Source ref: `{report['source_ref']}`\n\n" + f"- {changes['additions']} additions\n" + f"- {changes['updates']} updates\n" + f"- {changes['deletions']} deletions\n\n" + "The workflow rebuilt all reward mappings, resolved English names through " + "`Localization/en/game.json`, and passed the complete test suite. Pool mapping " + "files and UID inputs were not changed.\n\n" + f"[Workflow run]({os.environ['RUN_URL']})\n" + ) + Path("build/mapping-update/pull-request-body.md").write_text(body, encoding="utf-8") + PY + + pr_number="$(gh pr list \ + --head "$AUTOMATION_BRANCH" \ + --base "$BASE_BRANCH" \ + --state open \ + --json number \ + --jq '.[0].number // empty')" + + if [ -n "$pr_number" ]; then + gh pr edit "$pr_number" \ + --title "chore: update reward mappings from NTE_Assets" \ + --body-file build/mapping-update/pull-request-body.md + else + gh pr create \ + --head "$AUTOMATION_BRANCH" \ + --base "$BASE_BRANCH" \ + --title "chore: update reward mappings from NTE_Assets" \ + --body-file build/mapping-update/pull-request-body.md + fi diff --git a/README.md b/README.md index 7a50127..2ffd974 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,26 @@ The exporter automatically includes the shareable NTE user UID when it appears i > [!TIP] > For reliable deduplication, start from page 1 and scroll through the pages. If you only want pages 1–5, scroll through to page 6 as well just to be on the safe side. +## Mapping maintenance + +Reward metadata can be rebuilt directly from the latest NTE_Assets tables and +English translation files. The reward snapshot may change IDs while UID inputs +remain untouched: + +Run **Update reward mappings** from the GitHub Actions tab to generate and test +the snapshot in a reviewable pull request, or run it locally: + +```powershell +python tools/update_mappings.py +``` + +This rebuilds reviewable reward-map candidates directly from NTE_Assets under +`build/mapping-update/`; committed mappings are untouched unless `--apply` is +explicitly supplied. The snapshot may include additions, updates, and removals, +while pool mappings and UID inputs remain untouched. See +[Reward mapping updates](docs/mapping-updates.md) for the source rules and +review workflow. + ## Privacy > [!CAUTION] diff --git a/docs/mapping-updates.md b/docs/mapping-updates.md new file mode 100644 index 0000000..9978af9 --- /dev/null +++ b/docs/mapping-updates.md @@ -0,0 +1,95 @@ +# Reward mapping updates + +`tools/update_mappings.py` rebuilds `arcs.json`, `characters.json`, and +`items.json` directly from the current +[`Waifus-Grace/NTE_Assets`](https://github.com/Waifus-Grace/NTE_Assets) +tables. No other exporter or intermediate mapping format is used. + +English display names come from `Localization/en/game.json`. The data tables +provide localization namespace/key references and quality metadata; their +embedded `LocalizedString` values are deliberately ignored. + +The generated files are an authoritative snapshot, not an additive merge. +When NTE_Assets removes an ID, changes its casing, or updates its metadata, the +staged mapping reflects that change. This is intentional for a live-service +game whose asset tables evolve over time. + +## Review workflow + +### GitHub Actions + +Run **Update reward mappings** from the repository's Actions tab. The optional +`source_ref` input accepts an NTE_Assets branch, tag, or commit and defaults to +`main`. + +The workflow rebuilds the mappings, runs the complete test suite, uploads the +JSON review report, and opens a pull request from +`dev/automated-mapping-update`. Running it again updates the same open pull +request. If the authoritative snapshot has not changed, it reports zero changes +and does not create a commit or pull request. + +The repository setting **Allow GitHub Actions to create and approve pull +requests** must be enabled for automatic pull-request creation. The workflow +uses only the repository-scoped `GITHUB_TOKEN` and grants it `contents: write` +and `pull-requests: write` permissions. + +### Local + +Stage the latest candidate files without touching the committed mappings: + +```powershell +python tools/update_mappings.py +``` + +The required JSON tables are downloaded from the `main` branch by default. +Pin a tag, branch, or commit for reproducible review: + +```powershell +python tools/update_mappings.py --source-ref e9752e5963103529f0d683fd0aed4752b7dfad78 +``` + +To use an existing local checkout without network access: + +```powershell +python tools/update_mappings.py --assets-root path\to\NTE_Assets +``` + +The default output is `build/mapping-update/` and contains the three candidate +mapping files plus `mapping-update-report.json`. The report lists additions, +updates, and deletions by file and records SHA-256 hashes for every source +table. + +After reviewing the report and diff, apply the validated snapshot explicitly: + +```powershell +python tools/update_mappings.py --apply +python -m pytest -q +``` + +For CI, `--check` exits with status 1 whenever the generated snapshot differs +from the committed reward mappings. It still writes the staged artifacts. + +## Source rules + +- Every Arc in `DT_ForkItemData` is emitted to `arcs.json`. +- Every character in `DT_Character` is emitted to `characters.json`. +- Other pull rewards are selected by `GachaIllustrate` and resolved to the + inventory tables for quality and localization keys. Appearance-table IDs + provide canonical casing for glider rewards. +- Names are resolved strictly through `Localization/en/game.json`. A missing or + ambiguous key fails the update instead of falling back to a DT value. +- Character-awakening illustration entries are not independent pull rewards + and are excluded. +- Orange, purple, and blue item qualities map to `S`, `A`, and `B`. + +## UID compatibility boundary + +This updater reads and writes only the three reward mapping files. It never +reads or writes the permanent, limited, beginner, or Arc pool mapping files. +Banner IDs, timestamps, record ordering, format version, and all UID inputs are +therefore unchanged by mapping synchronization. + +Reward IDs and display metadata may change or disappear when the authoritative +NTE_Assets snapshot changes. Existing exported history files remain unchanged; +new exports describe rewards using the current asset snapshot. UID stability is +independent of reward display mappings. diff --git a/mappings/arcs.json b/mappings/arcs.json index c31ea08..1a23a05 100644 --- a/mappings/arcs.json +++ b/mappings/arcs.json @@ -1,38 +1,36 @@ { - "fork_dustbin": {"name": "Dangerous Game", "rank": "B"}, - "fork_vine": {"name": "Be Happy", "rank": "B"}, - "fork_nonos": {"name": "First Step to Success", "rank": "B"}, "fork_appliance": {"name": "\"Real Music\"", "rank": "B"}, - "fork_Prokaryon": {"name": "Us.", "rank": "B"}, - "fork_PaperPlane": {"name": "Clear Skies", "rank": "A"}, - "fork_mofeikesi": {"name": "Good Boy's Grand Adventure", "rank": "S"}, - "fork_wuhuakuang": {"name": "The Forgotten", "rank": "A"}, - "fork_whuakuang": {"name": "The Forgotten", "rank": "A"}, - "fork_jingmotingyuan": {"name": "Camellia Society", "rank": "S"}, - "fork_Kite": {"name": "Watch Your Heads!", "rank": "A"}, - "fork_yuren": {"name": "Umbrella", "rank": "A"}, - "fork_yaodao": {"name": "Drawn Blade", "rank": "A"}, "fork_Arachne": {"name": "Eternal Waltz", "rank": "S"}, "fork_BitGame": {"name": "Blow up the Crowd", "rank": "S"}, "fork_BitterCake": {"name": "The Good, The Bad, The Bitter", "rank": "A"}, "fork_BlackBook": {"name": "Youthful Fantasy", "rank": "S"}, - "fork_Baozhatang": {"name": "Fluff of Fearlessness", "rank": "S"}, + "fork_BlastCandy": {"name": "Fluff of Fearlessness", "rank": "S"}, "fork_bopu": {"name": "Cosmos Daze, Wild Reverie", "rank": "A"}, - "fork_Quanjitang": {"name": "Fluff of Fortitude", "rank": "S"}, + "fork_BoxingCandy": {"name": "Fluff of Fortitude", "rank": "S"}, "fork_Butterfly": {"name": "Reality Refuge", "rank": "S"}, "fork_Castle": {"name": "Call of the Twisted City", "rank": "A"}, - "fork_Qiaoqiao": {"name": "Time Bandit", "rank": "A"}, + "fork_Crowbar": {"name": "Time Bandit", "rank": "A"}, + "fork_Door": {"name": "The Wrong Gate", "rank": "S"}, + "fork_dustbin": {"name": "Dangerous Game", "rank": "B"}, + "fork_GoldWool": {"name": "What's Desired", "rank": "S"}, "fork_jiaojuan": {"name": "Shiny Days", "rank": "A"}, + "fork_jingmotingyuan": {"name": "Camellia Society", "rank": "S"}, + "fork_Kite": {"name": "Watch Your Heads!", "rank": "A"}, "fork_KnightCandy": {"name": "Fluff of Ferocity", "rank": "S"}, "fork_koinobori": {"name": "A Time Will Come", "rank": "A"}, "fork_lingganzhongjiezhe": {"name": "Mind Royale", "rank": "A"}, + "fork_LunarPhase": {"name": "Blushing Mirage", "rank": "S"}, "fork_mamen": {"name": "Contemplative Cat", "rank": "S"}, + "fork_mofeikesi": {"name": "Good Boy's Grand Adventure", "rank": "S"}, "fork_moon": {"name": "Stellar Veil", "rank": "S"}, "fork_MotorCandy": {"name": "Fluff of Fleetness", "rank": "S"}, "fork_Nakupeda": {"name": "Your Happiness is Priceless", "rank": "S"}, "fork_NestBird": {"name": "Tears Beneath the Mask", "rank": "S"}, + "fork_nonos": {"name": "First Step to Success", "rank": "B"}, "fork_oulaquantao": {"name": "Oraora!", "rank": "A"}, + "fork_PaperPlane": {"name": "Clear Skies", "rank": "A"}, "fork_PoliceRat": {"name": "Hethereau's Keeper", "rank": "S"}, + "fork_Prokaryon": {"name": "Us.", "rank": "B"}, "fork_rishi": {"name": "Day Off", "rank": "S"}, "fork_Rose": {"name": "The Last Rose", "rank": "S"}, "fork_snowman": {"name": "The Fools' Spring", "rank": "A"}, @@ -41,9 +39,11 @@ "fork_TigerTally": {"name": "Ready-Ready", "rank": "S"}, "fork_Time": {"name": "Marching Beyond Time", "rank": "S"}, "fork_tuansanlang": {"name": "The Great Thief", "rank": "A"}, + "fork_vine": {"name": "Be Happy", "rank": "B"}, "fork_Whale": {"name": "Song of the Whale", "rank": "S"}, "fork_worldrain": {"name": "The Rain That Shook the World", "rank": "S"}, - "fork_Wushoutieyu": {"name": "Raging Flames", "rank": "S"}, - "fork_GoldWool": {"name": "What's Desired", "rank": "S"}, - "fork_LunarPhase": {"name": "Blushing Mirage", "rank": "S"} + "fork_wuhuakuang": {"name": "The Forgotten", "rank": "A"}, + "fork_wushoutieyu": {"name": "Raging Flames", "rank": "S"}, + "fork_yaodao": {"name": "Drawn Blade", "rank": "A"}, + "fork_yuren": {"name": "Umbrella", "rank": "A"} } diff --git a/mappings/characters.json b/mappings/characters.json index 842b603..b037890 100644 --- a/mappings/characters.json +++ b/mappings/characters.json @@ -19,5 +19,6 @@ "1070": {"name": "Aurelia", "rank": "A"}, "1071": {"name": "Chaos", "rank": "S"}, "1073": {"name": "Chiz", "rank": "S"}, + "1075": {"name": "Iroi", "rank": "S"}, "1076": {"name": "Shinku", "rank": "S"} } diff --git a/mappings/items.json b/mappings/items.json index c795db2..f8daf7e 100644 --- a/mappings/items.json +++ b/mappings/items.json @@ -1,26 +1,25 @@ { - "DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"}, "Dice_ticket_01": {"type": "item", "name": "Warp Piece", "rank": "S"}, "Dice_ticket_02": {"type": "item", "name": "Lost Piece", "rank": "A"}, "Dicelimite": {"type": "item", "name": "Solid Dice", "rank": "S"}, - - "Fashion_Glide_1010": {"type": "cosmetic", "name": "Glider - Underboss-of-the-Underboss", "rank": "A"}, - "Fashion_vehicle_1010_V008": {"type": "cosmetic", "name": "Mod Parts - Tiger Incoming! - Livery", "rank": "S"}, - "Fashion_character_1010": {"type": "cosmetic", "name": "Character Skin - Phoenix Kick", "rank": "S"}, - - "Fashion_Glide_1052": {"type": "cosmetic", "name": "Glider - Orchid Breeze", "rank": "A"}, - "Fashion_vehicle_1052_V024": {"type": "cosmetic", "name": "Mod Parts - Autumn Haze - Livery", "rank": "S"}, - "Fashion_character_1052_01": {"type": "cosmetic", "name": "Character Skin - Priceless Orchid", "rank": "S"}, - - "Fashion_Glide_1004": {"type": "cosmetic", "name": "Glider - Tomato Duo", "rank": "A"}, - "Fashion_vehicle_1004_V021": {"type": "cosmetic", "name": "Mod Parts - Tomato Cruise - Livery", "rank": "S"}, - "Fashion_character_1004_01": {"type": "cosmetic", "name": "Character Skin - Gilded Rhapsody", "rank": "S"}, - - "Fashion_Glide_1071": {"type": "cosmetic", "name": "Glider - Skyrider", "rank": "A"}, - "Fashion_vehicle_1071_V010": {"type": "cosmetic", "name": "Mod Parts - Novis ST-X 950 - Livery", "rank": "S"}, - "Fashion_character_1071_01": {"type": "cosmetic", "name": "Character Skin - Clear Skies", "rank": "S"}, - - "Fashion_Glide_1076": {"type": "cosmetic", "name": "Glider - Overcast Canopy", "rank": "A"}, - "Fashion_vehicle_1076_V024": {"type": "cosmetic", "name": "Mod Parts - Hidden Dragon - Livery", "rank": "S"}, - "Fashion_character_1076_01": {"type": "cosmetic", "name": "Character Skin - Student of Terrasea", "rank": "S"} + "DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"}, + "Fashion_character_1004_01": {"type": "cosmetic", "name": "Gilded Rhapsody", "rank": "S"}, + "Fashion_character_1010": {"type": "cosmetic", "name": "Phoenix Kick", "rank": "S"}, + "Fashion_character_1052_01": {"type": "cosmetic", "name": "Priceless Orchid", "rank": "S"}, + "Fashion_character_1071_01": {"type": "cosmetic", "name": "Clear Skies", "rank": "S"}, + "Fashion_character_1075_01": {"type": "cosmetic", "name": "Summer Dream", "rank": "S"}, + "Fashion_character_1076_01": {"type": "cosmetic", "name": "Student of Terrasea", "rank": "S"}, + "Fashion_Glide_1004": {"type": "cosmetic", "name": "Tomato Duo", "rank": "A"}, + "Fashion_Glide_1010": {"type": "cosmetic", "name": "Underboss-of-the-Underboss", "rank": "A"}, + "Fashion_Glide_1052": {"type": "cosmetic", "name": "Orchid Breeze", "rank": "A"}, + "Fashion_Glide_1071": {"type": "cosmetic", "name": "Skyrider", "rank": "A"}, + "Fashion_glide_1075": {"type": "cosmetic", "name": "Skyrider", "rank": "A"}, + "Fashion_Glide_1076": {"type": "cosmetic", "name": "Overcast Canopy", "rank": "A"}, + "Fashion_vehicle_1004_V021": {"type": "cosmetic", "name": "Tomato Cruise - Livery", "rank": "S"}, + "Fashion_vehicle_1010_V008": {"type": "cosmetic", "name": "Tiger Incoming! - Livery", "rank": "S"}, + "Fashion_vehicle_1052_V024": {"type": "cosmetic", "name": "Autumn Haze - Livery", "rank": "S"}, + "Fashion_vehicle_1071_V010": {"type": "cosmetic", "name": "Hound Blitz - Livery", "rank": "S"}, + "Fashion_vehicle_1075_V006": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"}, + "Fashion_vehicle_1076_V024": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"}, + "Fashion_vehicleSkin_1052_V016": {"type": "cosmetic", "name": "Golden Breeze - Livery", "rank": "S"} } diff --git a/src/nte_history_exporter/mapping_update.py b/src/nte_history_exporter/mapping_update.py new file mode 100644 index 0000000..e231c34 --- /dev/null +++ b/src/nte_history_exporter/mapping_update.py @@ -0,0 +1,387 @@ +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +NTE_ASSETS_REPOSITORY = "https://github.com/Waifus-Grace/NTE_Assets" +DEFAULT_SOURCE_REF = "main" +ASSET_TABLES = { + "characters": "DataTable/Character/DT_Character.json", + "arcs": "DataTable/Fork/DT_ForkItemData.json", + "inventory": "DataTable/Inventory/DT_ItemConfig.json", + "capital_inventory": "DataTable/Inventory/DT_CapitalItemConfig.json", + "appearances": "DataTable/Character/Appearance/DT_AppearanceData.json", + "illustrations": "DataTable/Gacha/GachaIllustrate.json", + "localization": "Localization/en/game.json", +} +REWARD_MAPPING_FILES = ("arcs.json", "characters.json", "items.json") +RANK_BY_QUALITY = { + "EItemQuality::ITEM_QUALITY_ORANGE": "S", + "EItemQuality::ITEM_QUALITY_PURPLE": "A", + "EItemQuality::ITEM_QUALITY_BLUE": "B", +} + + +class MappingUpdateError(ValueError): + pass + + +@dataclass(frozen=True) +class AssetSource: + tables: dict[str, dict[str, Any]] + source: str + source_ref: str | None + sha256: str + file_sha256: dict[str, str] + + +@dataclass(frozen=True) +class MappingUpdateResult: + mappings: dict[str, dict[str, dict[str, Any]]] + report: dict[str, Any] + + @property + def change_count(self) -> int: + changes = self.report["changes"] + return changes["additions"] + changes["updates"] + changes["deletions"] + + +def load_assets(*, assets_root: Path | None = None, source_ref: str = DEFAULT_SOURCE_REF) -> AssetSource: + tables: dict[str, dict[str, Any]] = {} + file_hashes: dict[str, str] = {} + combined = hashlib.sha256() + + for label, relative_path in ASSET_TABLES.items(): + if assets_root is not None: + path = assets_root / Path(relative_path) + try: + raw = path.read_bytes() + except OSError as exc: + raise MappingUpdateError(f"cannot read NTE_Assets table {path}: {exc}") from exc + else: + url = f"https://raw.githubusercontent.com/Waifus-Grace/NTE_Assets/{source_ref}/{relative_path}" + request = urllib.request.Request(url, headers={"User-Agent": "nte-history-exporter-mapping-update"}) + try: + with urllib.request.urlopen(request, timeout=60) as response: + raw = response.read() + except OSError as exc: + raise MappingUpdateError(f"cannot download NTE_Assets table {relative_path}: {exc}") from exc + + try: + document = json.loads(raw.decode("utf-8-sig")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise MappingUpdateError(f"invalid NTE_Assets JSON in {relative_path}: {exc}") from exc + tables[label] = ( + _extract_localization(document, relative_path) + if label == "localization" + else _extract_rows(document, relative_path) + ) + digest = hashlib.sha256(raw).hexdigest() + file_hashes[relative_path] = digest + combined.update(relative_path.encode("utf-8")) + combined.update(b"\0") + combined.update(raw) + + source = str(assets_root.resolve()) if assets_root is not None else NTE_ASSETS_REPOSITORY + return AssetSource( + tables=tables, + source=source, + source_ref=None if assets_root is not None else source_ref, + sha256=combined.hexdigest(), + file_sha256=file_hashes, + ) + + +def load_current_mappings(directory: Path) -> dict[str, dict[str, dict[str, Any]]]: + mappings = {} + for filename in REWARD_MAPPING_FILES: + path = directory / filename + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise MappingUpdateError(f"cannot read {path}: {exc}") from exc + if not isinstance(value, dict): + raise MappingUpdateError(f"{path} must contain an object") + mappings[filename] = value + validate_mappings(mappings) + return mappings + + +def build_mapping_update( + current: dict[str, dict[str, dict[str, Any]]], + assets: AssetSource, +) -> MappingUpdateResult: + validate_mappings(current) + translations = _build_translation_index(assets.tables["localization"]) + output = { + "arcs.json": _build_primary_mapping(assets.tables["arcs"], "arc", translations), + "characters.json": _build_primary_mapping(assets.tables["characters"], "character", translations), + "items.json": _build_item_mapping(assets.tables, translations), + } + validate_mappings(output) + + additions: dict[str, list[str]] = {} + updates: dict[str, list[str]] = {} + deletions: dict[str, list[str]] = {} + for filename in REWARD_MAPPING_FILES: + old = current[filename] + new = output[filename] + additions[filename] = sorted(new.keys() - old.keys(), key=str.casefold) + deletions[filename] = sorted(old.keys() - new.keys(), key=str.casefold) + updates[filename] = sorted( + (item_id for item_id in old.keys() & new.keys() if old[item_id] != new[item_id]), + key=str.casefold, + ) + + report = { + "schema_version": 2, + "source": assets.source, + "source_ref": assets.source_ref, + "source_sha256": assets.sha256, + "source_file_sha256": assets.file_sha256, + "safety": { + "authoritative_snapshot": True, + "deletions_allowed": True, + "pool_mappings_touched": False, + "uid_inputs_touched": False, + }, + "changes": { + "additions": sum(map(len, additions.values())), + "updates": sum(map(len, updates.values())), + "deletions": sum(map(len, deletions.values())), + }, + "additions_by_file": additions, + "updates_by_file": updates, + "deletions_by_file": deletions, + "output_counts": {filename: len(entries) for filename, entries in output.items()}, + } + return MappingUpdateResult(output, report) + + +def write_update(result: MappingUpdateResult, directory: Path) -> None: + directory.mkdir(parents=True, exist_ok=True) + for filename in REWARD_MAPPING_FILES: + _atomic_write(directory / filename, _dump_mapping(result.mappings[filename])) + report_text = json.dumps(result.report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + _atomic_write(directory / "mapping-update-report.json", report_text) + + +def apply_update(result: MappingUpdateResult, directory: Path) -> None: + """Replace all reward maps as one transaction, rolling back on failure.""" + directory.mkdir(parents=True, exist_ok=True) + originals: dict[Path, bytes | None] = {} + written: list[Path] = [] + for filename in REWARD_MAPPING_FILES: + path = directory / filename + originals[path] = path.read_bytes() if path.exists() else None + try: + for filename in REWARD_MAPPING_FILES: + path = directory / filename + _atomic_write(path, _dump_mapping(result.mappings[filename])) + written.append(path) + except OSError: + for path in reversed(written): + original = originals[path] + if original is None: + path.unlink(missing_ok=True) + else: + _atomic_write_bytes(path, original) + raise + + +def validate_mappings(mappings: dict[str, dict[str, dict[str, Any]]]) -> None: + if set(mappings) != set(REWARD_MAPPING_FILES): + raise MappingUpdateError(f"mapping set must be exactly {REWARD_MAPPING_FILES}") + locations: dict[str, str] = {} + folded: dict[str, str] = {} + for filename in REWARD_MAPPING_FILES: + entries = mappings[filename] + if not isinstance(entries, dict): + raise MappingUpdateError(f"{filename} must contain an object") + for item_id, meta in entries.items(): + if not isinstance(item_id, str) or not item_id: + raise MappingUpdateError(f"{filename} contains an invalid item ID") + if item_id in locations: + raise MappingUpdateError(f"item ID {item_id} appears in {locations[item_id]} and {filename}") + locations[item_id] = filename + case_key = item_id.casefold() + if case_key in folded and folded[case_key] != item_id: + raise MappingUpdateError(f"case-insensitive duplicate IDs: {folded[case_key]} and {item_id}") + folded[case_key] = item_id + if not isinstance(meta, dict): + raise MappingUpdateError(f"{filename}:{item_id} must be an object") + name = meta.get("name") + if not isinstance(name, str) or not name.strip(): + raise MappingUpdateError(f"{filename}:{item_id}.name must be a non-empty string") + rank = meta.get("rank") + if filename == "arcs.json" and rank not in {"S", "A", "B"}: + raise MappingUpdateError(f"{filename}:{item_id}.rank must be S, A, or B") + if filename == "characters.json" and rank not in {"S", "A"}: + raise MappingUpdateError(f"{filename}:{item_id}.rank must be S or A") + if filename == "items.json": + if meta.get("type") not in {"item", "cosmetic"}: + raise MappingUpdateError(f"{filename}:{item_id}.type must be item or cosmetic") + if rank not in {"S", "A", "B"}: + raise MappingUpdateError(f"{filename}:{item_id}.rank must be S, A, or B") + + +def _extract_rows(document: Any, relative_path: str) -> dict[str, Any]: + candidates = document if isinstance(document, list) else [document] + for candidate in candidates: + if not isinstance(candidate, dict): + continue + rows = candidate.get("Rows") + if isinstance(rows, dict): + return rows + if all(isinstance(key, str) for key in candidate) and candidate: + return candidate + raise MappingUpdateError(f"NTE_Assets table {relative_path} does not contain a Rows object") + + +def _extract_localization(document: Any, relative_path: str) -> dict[str, Any]: + if not isinstance(document, dict) or not all(isinstance(value, dict) for value in document.values()): + raise MappingUpdateError(f"NTE_Assets localization {relative_path} must contain namespace objects") + return document + + +def _build_translation_index(localization: dict[str, Any]) -> dict[str, list[tuple[str, str]]]: + result: dict[str, list[tuple[str, str]]] = {} + for namespace, entries in localization.items(): + for key, value in entries.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise MappingUpdateError(f"invalid English localization entry {namespace}:{key}") + result.setdefault(key, []).append((namespace, value)) + return result + + +def _build_primary_mapping( + rows: dict[str, Any], + kind: str, + translations: dict[str, list[tuple[str, str]]], +) -> dict[str, dict[str, Any]]: + result = {} + for item_id in sorted(rows, key=lambda value: (value.casefold(), value)): + meta = _normalise_row(item_id, rows[item_id], translations) + if kind == "character" and meta["rank"] not in {"S", "A"}: + raise MappingUpdateError(f"unexpected character quality for {item_id}") + result[item_id] = {"name": meta["name"], "rank": meta["rank"]} + return result + + +def _build_item_mapping( + tables: dict[str, dict[str, Any]], + translations: dict[str, list[tuple[str, str]]], +) -> dict[str, dict[str, Any]]: + inventory = _casefold_index(tables["inventory"], "inventory") + capital = _casefold_index(tables["capital_inventory"], "capital inventory") + appearances = _casefold_index(tables["appearances"], "appearances") + result: dict[str, dict[str, Any]] = {} + for illustrated_id in tables["illustrations"]: + folded = illustrated_id.casefold() + if illustrated_id.isdigit() or folded.startswith("fork_") or folded.startswith("characterawaken_"): + continue + sources = (inventory, capital, appearances) + match = next((source.get(folded) for source in sources if folded in source), None) + if match is None: + raise MappingUpdateError(f"illustrated reward {illustrated_id} is missing from item tables") + canonical_id, row = match + if folded.startswith("fashion_glide_") and folded in appearances: + canonical_id = appearances[folded][0] + meta = _normalise_row(canonical_id, row, translations) + result[canonical_id] = { + "type": "cosmetic" if canonical_id.casefold().startswith("fashion_") else "item", + "name": meta["name"], + "rank": meta["rank"], + } + return dict(sorted(result.items(), key=lambda pair: (pair[0].casefold(), pair[0]))) + + +def _casefold_index(rows: dict[str, Any], label: str) -> dict[str, tuple[str, Any]]: + result: dict[str, tuple[str, Any]] = {} + for item_id, row in rows.items(): + folded = item_id.casefold() + if folded in result and result[folded][0] != item_id: + raise MappingUpdateError(f"{label} has case-insensitive duplicate IDs: {result[folded][0]} and {item_id}") + result[folded] = (item_id, row) + return result + + +def _normalise_row( + item_id: str, + row: Any, + translations: dict[str, list[tuple[str, str]]], +) -> dict[str, str]: + if not isinstance(row, dict): + raise MappingUpdateError(f"NTE_Assets row {item_id} must be an object") + item_name = row.get("ItemName") or row.get("Name") + if not isinstance(item_name, dict): + raise MappingUpdateError(f"NTE_Assets row {item_id} has no ItemName") + name = _translate_name(item_id, item_name, translations) + quality = row.get("ItemQuality") or row.get("Quality") + rank = RANK_BY_QUALITY.get(quality) + if rank is None: + raise MappingUpdateError(f"NTE_Assets row {item_id} has unsupported quality {quality!r}") + return {"name": name.strip(), "rank": rank} + + +def _translate_name( + item_id: str, + string_reference: dict[str, Any], + translations: dict[str, list[tuple[str, str]]], +) -> str: + key = string_reference.get("Key") + if not isinstance(key, str) or not key: + raise MappingUpdateError(f"NTE_Assets row {item_id} has no localization key") + table_id = string_reference.get("TableId") + expected_namespace = None + if isinstance(table_id, str) and table_id: + expected_namespace = table_id.rsplit("/", 1)[-1].split(".", 1)[0] + matches = translations.get(key, []) + preferred = [value for namespace, value in matches if namespace == expected_namespace] + if len(preferred) == 1: + name = preferred[0] + elif len(matches) == 1: + name = matches[0][1] + elif not matches: + raise MappingUpdateError(f"English localization is missing {item_id} key {key}") + else: + namespaces = ", ".join(namespace for namespace, _value in matches) + raise MappingUpdateError(f"English localization key {key} for {item_id} is ambiguous: {namespaces}") + if not name.strip(): + raise MappingUpdateError(f"English localization key {key} for {item_id} is empty") + return name + + +def _dump_mapping(mapping: dict[str, dict[str, Any]]) -> str: + lines = ["{"] + entries = list(mapping.items()) + for index, (item_id, meta) in enumerate(entries): + comma = "," if index < len(entries) - 1 else "" + key = json.dumps(item_id, ensure_ascii=False) + value = json.dumps(meta, ensure_ascii=False, separators=(", ", ": ")) + lines.append(f" {key}: {value}{comma}") + lines.append("}") + return "\n".join(lines) + "\n" + + +def _atomic_write(path: Path, text: str) -> None: + _atomic_write_bytes(path, text.encode("utf-8")) + + +def _atomic_write_bytes(path: Path, content: bytes) -> None: + with tempfile.NamedTemporaryFile( + mode="wb", dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False + ) as handle: + handle.write(content) + temporary = Path(handle.name) + try: + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 7c54fd3..abdedc9 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -21,3 +21,12 @@ points gifts, chase rewards, batched pages, and bit-packed responses. Do not replace this file with a real `.pcap`, `.flows`, or exported account history. Add new cases by constructing the smallest relevant payload, replacing all timestamps and endpoints, and extending the privacy assertions. + +## Synthetic NTE_Assets fixture + +`nte_assets/` mirrors only the six table paths and English localization file +consumed by the mapping updater. Its rows, IDs, and names are deliberately +synthetic and contain no account or personal data. The fixture exercises +authoritative additions, metadata updates, deletions, translation lookup, +illustration filtering, and canonical ID casing without depending on the +network or copying a live game-data snapshot into the repository. diff --git a/tests/fixtures/nte_assets/DataTable/Character/Appearance/DT_AppearanceData.json b/tests/fixtures/nte_assets/DataTable/Character/Appearance/DT_AppearanceData.json new file mode 100644 index 0000000..d513855 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/Character/Appearance/DT_AppearanceData.json @@ -0,0 +1 @@ +[{"Rows":{"Fashion_Glide_2000":{"Name":{"TableId":"/Game/Text/ST_Appearance.ST_Appearance","Key":"glider_2000","LocalizedString":"Wrong Embedded Appearance"},"Quality":"EItemQuality::ITEM_QUALITY_PURPLE"}}}] diff --git a/tests/fixtures/nte_assets/DataTable/Character/DT_Character.json b/tests/fixtures/nte_assets/DataTable/Character/DT_Character.json new file mode 100644 index 0000000..9866018 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/Character/DT_Character.json @@ -0,0 +1 @@ +[{"Rows":{"1003":{"ItemName":{"TableId":"/Game/Text/ST_Player.ST_Player","Key":"character_1003","LocalizedString":"Wrong Embedded Character"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"},"1099":{"ItemName":{"TableId":"/Game/Text/ST_Player.ST_Player","Key":"character_1099","LocalizedString":"Wrong Embedded New Character"},"ItemQuality":"EItemQuality::ITEM_QUALITY_PURPLE"}}}] diff --git a/tests/fixtures/nte_assets/DataTable/Fork/DT_ForkItemData.json b/tests/fixtures/nte_assets/DataTable/Fork/DT_ForkItemData.json new file mode 100644 index 0000000..fe59f68 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/Fork/DT_ForkItemData.json @@ -0,0 +1 @@ +[{"Rows":{"fork_alpha":{"ItemName":{"TableId":"/Game/Text/ST_Fork.ST_Fork","Key":"fork_alpha_name","LocalizedString":"Wrong Embedded Alpha"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"},"fork_new":{"ItemName":{"TableId":"/Game/Text/ST_Fork.ST_Fork","Key":"fork_new_name","LocalizedString":"Wrong Embedded New Arc"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"}}}] diff --git a/tests/fixtures/nte_assets/DataTable/Gacha/GachaIllustrate.json b/tests/fixtures/nte_assets/DataTable/Gacha/GachaIllustrate.json new file mode 100644 index 0000000..bd2fdf3 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/Gacha/GachaIllustrate.json @@ -0,0 +1 @@ +[{"Rows":{"1003":{},"1099":{},"fork_alpha":{},"fork_new":{},"DIceNormal":{},"Dice_ticket_03":{},"Fashion_glide_2000":{},"Characterawaken_1003":{}}}] diff --git a/tests/fixtures/nte_assets/DataTable/Inventory/DT_CapitalItemConfig.json b/tests/fixtures/nte_assets/DataTable/Inventory/DT_CapitalItemConfig.json new file mode 100644 index 0000000..edf0e46 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/Inventory/DT_CapitalItemConfig.json @@ -0,0 +1 @@ +[{"Rows":{}}] diff --git a/tests/fixtures/nte_assets/DataTable/Inventory/DT_ItemConfig.json b/tests/fixtures/nte_assets/DataTable/Inventory/DT_ItemConfig.json new file mode 100644 index 0000000..a98e2f8 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/Inventory/DT_ItemConfig.json @@ -0,0 +1 @@ +[{"Rows":{"DiceNormal":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"dice_normal","LocalizedString":"Wrong Embedded Dice"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"},"Dice_ticket_03":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"ticket_03","LocalizedString":"Wrong Embedded Ticket"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"},"Fashion_glide_2000":{"ItemName":{"TableId":"/Game/Text/ST_Appearance.ST_Appearance","Key":"glider_2000","LocalizedString":"Wrong Embedded Glider"},"ItemQuality":"EItemQuality::ITEM_QUALITY_PURPLE"},"UnusedItem":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"unused_item","LocalizedString":"Wrong Embedded Unused"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"}}}] diff --git a/tests/fixtures/nte_assets/Localization/en/game.json b/tests/fixtures/nte_assets/Localization/en/game.json new file mode 100644 index 0000000..5bda83c --- /dev/null +++ b/tests/fixtures/nte_assets/Localization/en/game.json @@ -0,0 +1,18 @@ +{ + "ST_Player": { + "character_1003": "Old Character Renamed", + "character_1099": "New Character" + }, + "ST_Fork": { + "fork_alpha_name": "Alpha Arc", + "fork_new_name": "New Arc" + }, + "ST_Item": { + "dice_normal": "Fabricated Dice", + "ticket_03": "New Ticket", + "unused_item": "Not A Pull Reward" + }, + "ST_Appearance": { + "glider_2000": "Inventory Glider" + } +} diff --git a/tests/test_mapping_update.py b/tests/test_mapping_update.py new file mode 100644 index 0000000..a715f16 --- /dev/null +++ b/tests/test_mapping_update.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +import unittest +from copy import deepcopy +from dataclasses import replace +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from nte_history_exporter.mapping_update import ( + MappingUpdateError, + apply_update, + build_mapping_update, + load_assets, + load_current_mappings, + write_update, +) +from tests.support import FIXTURES + + +SAMPLE_ASSETS = FIXTURES / "nte_assets" + + +def sample_current(): + return { + "arcs.json": { + "fork_alpha": {"name": "Stale Alpha Name", "rank": "B"}, + "fork_retired": {"name": "Retired Arc", "rank": "A"}, + }, + "characters.json": { + "1003": {"name": "Stale Character Name", "rank": "S"}, + "1001": {"name": "Retired Character", "rank": "A"}, + }, + "items.json": { + "DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"}, + "OldTicket": {"type": "item", "name": "Retired Ticket", "rank": "B"}, + }, + } + + +def build_sample_update(): + current = sample_current() + return current, build_mapping_update(current, load_assets(assets_root=SAMPLE_ASSETS)) + + +class MappingUpdateTests(unittest.TestCase): + def test_mapping_update_is_an_authoritative_snapshot(self): + current, result = build_sample_update() + + self.assertEqual(result.mappings["characters.json"]["1003"], {"name": "Old Character Renamed", "rank": "S"}) + self.assertEqual(result.mappings["characters.json"]["1099"], {"name": "New Character", "rank": "A"}) + self.assertNotIn("1001", result.mappings["characters.json"]) + self.assertNotIn("fork_retired", result.mappings["arcs.json"]) + self.assertNotIn("OldTicket", result.mappings["items.json"]) + self.assertNotEqual(result.mappings["characters.json"]["1003"], current["characters.json"]["1003"]) + self.assertEqual(result.report["changes"], {"additions": 4, "updates": 2, "deletions": 3}) + self.assertIs(result.report["safety"]["authoritative_snapshot"], True) + self.assertIs(result.report["safety"]["deletions_allowed"], True) + self.assertIs(result.report["safety"]["uid_inputs_touched"], False) + self.assertIs(result.report["safety"]["pool_mappings_touched"], False) + + def test_items_use_illustrations_as_filter_and_asset_tables_as_authority(self): + _current, result = build_sample_update() + + self.assertEqual( + result.mappings["items.json"]["Fashion_Glide_2000"], + {"type": "cosmetic", "name": "Inventory Glider", "rank": "A"}, + ) + self.assertIn("DiceNormal", result.mappings["items.json"]) + self.assertNotIn("DIceNormal", result.mappings["items.json"]) + self.assertNotIn("UnusedItem", result.mappings["items.json"]) + self.assertFalse(any(key.startswith("Characterawaken_") for key in result.mappings["items.json"])) + + def test_embedded_localized_strings_are_ignored(self): + _current, result = build_sample_update() + + self.assertEqual(result.mappings["characters.json"]["1003"]["name"], "Old Character Renamed") + self.assertNotEqual(result.mappings["characters.json"]["1003"]["name"], "Wrong Embedded Character") + + def test_mapping_update_rejects_missing_english_translation(self): + assets = load_assets(assets_root=SAMPLE_ASSETS) + tables = deepcopy(assets.tables) + del tables["localization"]["ST_Player"]["character_1003"] + + with self.assertRaisesRegex(MappingUpdateError, "English localization is missing"): + build_mapping_update(sample_current(), replace(assets, tables=tables)) + + def test_mapping_update_rejects_ambiguous_translation_without_namespace_match(self): + assets = load_assets(assets_root=SAMPLE_ASSETS) + tables = deepcopy(assets.tables) + tables["inventory"]["Fashion_glide_2000"]["ItemName"]["TableId"] = "/Game/Text/Unknown.Unknown" + tables["localization"]["OtherNamespace"] = {"glider_2000": "Other Glider"} + + with self.assertRaisesRegex(MappingUpdateError, "is ambiguous"): + build_mapping_update(sample_current(), replace(assets, tables=tables)) + + def test_local_asset_source_has_reproducible_provenance(self): + first = load_assets(assets_root=SAMPLE_ASSETS) + second = load_assets(assets_root=SAMPLE_ASSETS) + + self.assertEqual(first.sha256, second.sha256) + self.assertEqual(first.file_sha256, second.file_sha256) + self.assertEqual(first.source, str(SAMPLE_ASSETS.resolve())) + self.assertIsNone(first.source_ref) + + def test_mapping_update_writes_deterministic_review_artifacts(self): + _current, result = build_sample_update() + with TemporaryDirectory() as first_tmp, TemporaryDirectory() as second_tmp: + first = Path(first_tmp) + second = Path(second_tmp) + write_update(result, first) + write_update(result, second) + + for filename in (*result.mappings, "mapping-update-report.json"): + self.assertEqual((first / filename).read_bytes(), (second / filename).read_bytes()) + report = json.loads((first / "mapping-update-report.json").read_text(encoding="utf-8")) + self.assertEqual(report["changes"]["deletions"], 3) + + def test_mapping_update_does_not_mutate_inputs(self): + current = sample_current() + original = deepcopy(current) + + build_mapping_update(current, load_assets(assets_root=SAMPLE_ASSETS)) + + self.assertEqual(current, original) + + def test_mapping_update_apply_touches_only_reward_mapping_files(self): + _current, result = build_sample_update() + with TemporaryDirectory() as tmp: + directory = Path(tmp) + for filename, mapping in sample_current().items(): + (directory / filename).write_text(json.dumps(mapping), encoding="utf-8") + protected = directory / "permanent_board.json" + protected.write_text('{"protected": true}\n', encoding="utf-8") + + apply_update(result, directory) + + self.assertEqual(load_current_mappings(directory), result.mappings) + self.assertEqual(protected.read_text(encoding="utf-8"), '{"protected": true}\n') + self.assertFalse((directory / "mapping-update-report.json").exists()) + + def test_mapping_update_apply_rolls_back_partial_write(self): + _current, result = build_sample_update() + with TemporaryDirectory() as tmp: + directory = Path(tmp) + originals = {} + for filename, mapping in sample_current().items(): + destination = directory / filename + destination.write_text(json.dumps(mapping), encoding="utf-8") + originals[filename] = destination.read_bytes() + + from nte_history_exporter import mapping_update + + real_write = mapping_update._atomic_write + calls = 0 + + def fail_second_write(path, text): + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("synthetic write failure") + return real_write(path, text) + + with ( + patch("nte_history_exporter.mapping_update._atomic_write", side_effect=fail_second_write), + self.assertRaisesRegex(OSError, "synthetic write failure"), + ): + apply_update(result, directory) + + for filename, original in originals.items(): + self.assertEqual((directory / filename).read_bytes(), original) + + def test_mapping_update_rejects_unresolved_illustrated_rewards(self): + assets = load_assets(assets_root=SAMPLE_ASSETS) + tables = deepcopy(assets.tables) + tables["illustrations"]["MissingReward"] = {} + + with self.assertRaisesRegex(MappingUpdateError, "MissingReward"): + build_mapping_update(sample_current(), replace(assets, tables=tables)) + + def test_load_assets_rejects_missing_tables(self): + with TemporaryDirectory() as tmp, self.assertRaisesRegex(MappingUpdateError, "cannot read"): + load_assets(assets_root=Path(tmp)) diff --git a/tools/update_mappings.py b/tools/update_mappings.py new file mode 100644 index 0000000..9d71a80 --- /dev/null +++ b/tools/update_mappings.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from nte_history_exporter.mapping_update import ( # noqa: E402 + DEFAULT_SOURCE_REF, + MappingUpdateError, + apply_update, + build_mapping_update, + load_assets, + load_current_mappings, + write_update, +) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Synchronise reward mappings from the authoritative NTE_Assets tables." + ) + parser.add_argument( + "--assets-root", + type=Path, + help="local NTE_Assets checkout (downloads the required tables when omitted)", + ) + parser.add_argument( + "--source-ref", + default=DEFAULT_SOURCE_REF, + help="NTE_Assets branch, tag, or commit to download (default: main)", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=ROOT / "build" / "mapping-update", + help="directory for staged mappings and the review report", + ) + parser.add_argument( + "--apply", + action="store_true", + help="replace reward mapping files after staging and validation", + ) + parser.add_argument( + "--check", + action="store_true", + help="exit 1 when the generated snapshot differs from local mappings", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + assets = load_assets(assets_root=args.assets_root, source_ref=args.source_ref) + current = load_current_mappings(ROOT / "mappings") + result = build_mapping_update(current, assets) + write_update(result, args.output_dir) + if args.apply: + apply_update(result, ROOT / "mappings") + except (MappingUpdateError, OSError) as exc: + print(f"mapping update failed: {exc}", file=sys.stderr) + return 2 + + changes = result.report["changes"] + print(f"source: {assets.source}") + if assets.source_ref: + print(f"source ref: {assets.source_ref}") + print(f"staged: {args.output_dir}") + print( + "changes: " + f"{changes['additions']} additions, " + f"{changes['updates']} updates, " + f"{changes['deletions']} deletions" + ) + if args.apply: + print("applied: mappings/arcs.json, mappings/characters.json, mappings/items.json") + return 1 if args.check and result.change_count else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a5312520e73efaecbab991bf8dbed24ee2ad0750 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Wed, 15 Jul 2026 19:32:40 +0100 Subject: [PATCH 3/8] Implemented fallback decoder Added a new fallback decoder on the chance things do not decode correctly with the primary decoder. --- .github/workflows/release.yml | 2 +- docs/limitations.md | 3 + docs/mapping-updates.md | 2 +- docs/packet-format.md | 24 ++ src/nte_history_exporter/decoder/arc.py | 65 +++- src/nte_history_exporter/decoder/protocol.py | 120 +++++++- .../decoder/structured_protocol.py | 278 ++++++++++++++++++ src/nte_history_exporter/export/csv_export.py | 5 + tests/test_structured_protocol.py | 197 +++++++++++++ 9 files changed, 691 insertions(+), 5 deletions(-) create mode 100644 src/nte_history_exporter/decoder/structured_protocol.py create mode 100644 tests/test_structured_protocol.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 37cd713..f80552c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -196,7 +196,7 @@ jobs: echo "$description" | sed 's/^/ /' fi echo - done < <(git rev-list "$commit_range") + done < <(git rev-list --no-merges "$commit_range") } > release-notes.md - name: Download Artifacts diff --git a/docs/limitations.md b/docs/limitations.md index ee80883..01c4a2b 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -10,6 +10,9 @@ Known limitations: - A partially captured oldest timestamp group is still exported; its captured prefix has stable UIDs, and a later deeper scan adds the rest with the same UIDs. - Pages are anchored to the continuous run starting at page 1. Live capture reports gaps immediately and accepts replacement pages from another pass while the exporter remains open. Scrolling backward does not request cached pages again, so close and reopen the history board before rescanning. Any gap left when capture ends causes later pages to be ignored and reported as warnings. - Pipelined page requests and multi-page responses are supported, including observed 10-record responses containing two consecutive Monopoly pages. Non-byte-aligned response payloads are realigned before decoding. +- Structured Monopoly and Arc blocks are parsed in enrichment/fallback mode. + The existing decoder remains authoritative when structured rows do not agree + on record count, reward ID, and timestamp. - Live capture prefers Npcap on Windows and automatically falls back to the built-in raw-socket backend if Npcap is unavailable. Linux and macOS require the system libpcap runtime. - The file adapter reads mitmproxy `.flows` captures for research and testing. - Npcap is Windows-only and is not redistributed with this project; Linux and macOS use their system libpcap. diff --git a/docs/mapping-updates.md b/docs/mapping-updates.md index 9978af9..e4212e7 100644 --- a/docs/mapping-updates.md +++ b/docs/mapping-updates.md @@ -3,7 +3,7 @@ `tools/update_mappings.py` rebuilds `arcs.json`, `characters.json`, and `items.json` directly from the current [`Waifus-Grace/NTE_Assets`](https://github.com/Waifus-Grace/NTE_Assets) -tables. No other exporter or intermediate mapping format is used. +tables. English display names come from `Localization/en/game.json`. The data tables provide localization namespace/key references and quality metadata; their diff --git a/docs/packet-format.md b/docs/packet-format.md index dacc8de..50d7a74 100644 --- a/docs/packet-format.md +++ b/docs/packet-format.md @@ -2,6 +2,30 @@ This prototype supports separate Monopoly and Arc/Gashapon history decoders. +## Decoder strategy + +Both history paths now recognize the structured `FMonopolyLotteryRecordData` +and `FForkLotteryRecordData` blocks, including blocks packed at a non-byte +alignment. Structured fields include the item/count pair, pool ID, secondary +reward data, roll result, and standard .NET timestamp. + +The structured parser is deliberately compatibility-gated: + +- The established decoder remains the primary path. +- Structured rows enrich primary rows only when row count, reward ID, and raw + timestamp agree. Enrichment supplies exact quantities, missing roll details, + pool diagnostics, and secondary reward diagnostics. +- If the primary decoder returns no records but a complete structured block is + valid, the structured records are converted into the same internal row shape + as a fallback. +- Malformed, incomplete, mismatched, or ambiguous structured data is ignored; + it cannot overwrite a successfully decoded primary row. + +`decoder_mode`, `structured_protocol_view`, `structured_pool_id`, +`secondary_reward_id`, and `secondary_quantity` are research/debug CSV fields. +They are intentionally omitted from the public JSON export, whose format stays +at version 1. + ## Monopoly - History is fetched over the UDP game connection. diff --git a/src/nte_history_exporter/decoder/arc.py b/src/nte_history_exporter/decoder/arc.py index c353664..9f1bdc7 100644 --- a/src/nte_history_exporter/decoder/arc.py +++ b/src/nte_history_exporter/decoder/arc.py @@ -22,6 +22,7 @@ from nte_history_exporter.constants import ( from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 from nte_history_exporter.decoder.run import fmt_packet_time from nte_history_exporter.mappings import ARC_META +from nte_history_exporter.decoder.structured_protocol import StructuredRecord, parse_structured_records def is_arc_history_request(content: bytes) -> bool: @@ -61,7 +62,7 @@ def decode_arc_timestamp(raw8: bytes) -> tuple[int, float, str]: return ticks, unix_seconds, decoded -def parse_arc_response(response: bytes) -> list[dict[str, Any]]: +def _parse_legacy_arc_response(response: bytes) -> list[dict[str, Any]]: pos = ARC_RESPONSE_FIRST_RECORD_OFFSET records: list[dict[str, Any]] = [] while pos + 4 <= len(response): @@ -116,6 +117,68 @@ def parse_arc_response(response: bytes) -> list[dict[str, Any]]: return records +def _arc_metadata(arc_id: str) -> dict[str, Any]: + direct = ARC_META.get(arc_id) + if direct is not None: + return direct + folded = arc_id.casefold() + return next((meta for item_id, meta in ARC_META.items() if item_id.casefold() == folded), {}) + + +def _structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]: + records = [] + for structured in structured_rows: + meta = _arc_metadata(structured.item_id) + records.append( + { + "record_start": structured.record_start, + "record_end": structured.record_end, + "record_len": structured.record_end - structured.record_start, + "reward_key_hex": "", + "reward_type": "arc", + "reward_id": structured.item_id, + "reward_name": meta.get("name", "UNKNOWN"), + "reward_rank": meta.get("rank", ""), + "type_key_hex": "", + "source_type": "miracle_box", + "timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(), + "timestamp_ticks": structured.ticks, + "timestamp_unix": structured.timestamp_unix, + "timestamp_decoded": structured.timestamp_decoded, + "record_hex": structured.record_hex, + "decoder_mode": "structured_fallback", + "structured_pool_id": structured.pool_id, + "structured_protocol_view": structured.protocol_view, + } + ) + return records + + +def _enrich_legacy_arc_rows( + legacy_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord] +) -> list[dict[str, Any]]: + if len(legacy_rows) != len(structured_rows): + return legacy_rows + for legacy, structured in zip(legacy_rows, structured_rows): + if legacy.get("reward_id", "").casefold() != structured.item_id.casefold(): + return legacy_rows + if legacy.get("timestamp_ticks") not in {structured.ticks, structured.ticks * 2}: + return legacy_rows + for legacy, structured in zip(legacy_rows, structured_rows): + legacy["decoder_mode"] = "heuristic_enriched" + legacy["structured_pool_id"] = structured.pool_id + legacy["structured_protocol_view"] = structured.protocol_view + return legacy_rows + + +def parse_arc_response(response: bytes) -> list[dict[str, Any]]: + structured_rows = parse_structured_records(response, "fork") + legacy_rows = _parse_legacy_arc_response(response) + if legacy_rows: + return _enrich_legacy_arc_rows(legacy_rows, structured_rows) + return _structured_arc_rows(structured_rows) + + def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: pool = POOL_META["arc_miracle_box"] rows: list[dict[str, Any]] = [] diff --git a/src/nte_history_exporter/decoder/protocol.py b/src/nte_history_exporter/decoder/protocol.py index 6a40b56..c890257 100644 --- a/src/nte_history_exporter/decoder/protocol.py +++ b/src/nte_history_exporter/decoder/protocol.py @@ -16,6 +16,10 @@ from nte_history_exporter.constants import ( VALID_DICE_FIELDS, ) from nte_history_exporter.mappings import REWARDS_BY_ID +from nte_history_exporter.decoder.structured_protocol import ( + StructuredRecord, + parse_structured_records, +) REWARD_ID_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_") WARP_PIECE_CHASE_PATTERN = bytes.fromhex( @@ -299,7 +303,116 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str, return rows +def _structured_result(raw: int | None) -> tuple[int | None, int | None, str, int | None]: + if raw is None: + return None, None, "unknown", None + if raw == 0: + return 0, 0, "points_gift", 0 + if raw == 0xFFFFFFFF: + return -4, -4, "chase_reward", -4 + return raw, raw, "dice", raw + + +def _structured_rows_compatible( + heuristic_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord] +) -> bool: + if len(heuristic_rows) != len(structured_rows): + return False + for heuristic, structured in zip(heuristic_rows, structured_rows): + heuristic_id = heuristic.get("reward_id") or "" + if heuristic_id and heuristic_id.casefold() != structured.item_id.casefold(): + return False + heuristic_ticks = heuristic.get("timestamp_ticks") + if heuristic_ticks not in {structured.ticks, structured.ticks * 4}: + return False + return True + + +def _enrich_heuristic_rows( + heuristic_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord] +) -> list[dict[str, Any]]: + if not _structured_rows_compatible(heuristic_rows, structured_rows): + return heuristic_rows + for heuristic, structured in zip(heuristic_rows, structured_rows): + if not heuristic.get("reward_id"): + heuristic["reward_id"] = structured.item_id + reward = _reward_metadata(structured.item_id) + heuristic["reward_type"] = reward.get("type") or infer_reward_type(structured.item_id) + heuristic["reward_name"] = reward.get("name", "") + heuristic["reward_rank"] = reward.get("rank") + heuristic["quantity"] = structured.count + if heuristic.get("result_type") == "unknown" or heuristic.get("dice") is None: + dice, dice_raw, result_type, result_source = _structured_result(structured.roll_points_raw) + heuristic["dice"] = dice + heuristic["dice_raw_u32"] = dice_raw + heuristic["result_type"] = result_type + heuristic["result_source_raw"] = result_source + heuristic["roll_result"] = ( + "Points Gift" + if result_type == "points_gift" + else ("Chase Reward" if result_type == "chase_reward" else (f"Dice {dice}" if dice else "")) + ) + heuristic["decoder_mode"] = "heuristic_enriched" + heuristic["structured_pool_id"] = structured.pool_id + heuristic["secondary_reward_id"] = structured.secondary_item_id + heuristic["secondary_quantity"] = structured.secondary_count + heuristic["structured_protocol_view"] = structured.protocol_view + return heuristic_rows + + +def _reward_metadata(reward_id: str) -> dict[str, Any]: + direct = REWARDS_BY_ID.get(reward_id) + if direct is not None: + return direct + folded = reward_id.casefold() + return next((meta for item_id, meta in REWARDS_BY_ID.items() if item_id.casefold() == folded), {}) + + +def _structured_monopoly_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]: + rows = [] + for row_index, structured in enumerate(structured_rows, start=1): + dice, dice_raw, result_type, result_source = _structured_result(structured.roll_points_raw) + reward = _reward_metadata(structured.item_id) + rows.append( + { + "row": row_index, + "record_start": structured.record_start, + "record_end": structured.record_end, + "record_len": structured.record_end - structured.record_start, + "dice": dice, + "roll_result": ( + "Points Gift" + if result_type == "points_gift" + else ("Chase Reward" if result_type == "chase_reward" else (f"Dice {dice}" if dice else "")) + ), + "result_type": result_type, + "result_source_raw": result_source, + "dice_raw_u32": dice_raw, + "dice_offset_in_record": None, + "reward_key_hex": "", + "reward_type": reward.get("type") or infer_reward_type(structured.item_id), + "reward_id": structured.item_id, + "reward_name": reward.get("name", ""), + "reward_rank": reward.get("rank"), + "quantity": structured.count, + "timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(), + "timestamp_ticks": structured.ticks, + "timestamp_unix": f"{structured.timestamp_unix:.6f}", + "timestamp_decoded": structured.timestamp_decoded, + "record_hex": structured.record_hex, + "decoder_mode": "structured_fallback", + "structured_pool_id": structured.pool_id, + "secondary_reward_id": structured.secondary_item_id, + "secondary_quantity": structured.secondary_count, + "structured_protocol_view": structured.protocol_view, + } + ) + return rows + + def decode_response_records(response_content: bytes) -> list[dict[str, Any]]: + structured_rows = parse_structured_records(response_content, "monopoly") + heuristic_rows: list[dict[str, Any]] = [] for candidate in iter_history_response_alignments(response_content): if not any(marker in candidate for marker in MARKERS): continue @@ -308,5 +421,8 @@ def decode_response_records(response_content: bytes) -> list[dict[str, Any]]: except (OSError, OverflowError, ValueError): continue if rows: - return rows - return [] + heuristic_rows = rows + break + if heuristic_rows: + return _enrich_heuristic_rows(heuristic_rows, structured_rows) + return _structured_monopoly_rows(structured_rows) diff --git a/src/nte_history_exporter/decoder/structured_protocol.py b/src/nte_history_exporter/decoder/structured_protocol.py new file mode 100644 index 0000000..c541147 --- /dev/null +++ b/src/nte_history_exporter/decoder/structured_protocol.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import struct +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Literal + + +RecordType = Literal["monopoly", "fork"] + +MONOPOLY_MARKER = b"FMonopolyLotteryRecordData" +FORK_MARKER = b"FForkLotteryRecordData" +MAX_ROWS_PER_BLOCK = 100 +MAX_STRING_LENGTH = 256 +DOTNET_EPOCH_TICKS = 621_355_968_000_000_000 +DOTNET_TICKS_PER_SECOND = 10_000_000 +MIN_UNIX_SECONDS = 1_500_000_000 +MAX_UNIX_SECONDS = 4_102_444_800 + + +class StructuredProtocolError(ValueError): + pass + + +@dataclass(frozen=True) +class StructuredRecord: + record_type: RecordType + item_id: str + count: int + ticks: int + timestamp_unix: float + timestamp_decoded: str + pool_id: str | None + roll_points_raw: int | None + secondary_item_id: str | None + secondary_count: int | None + record_start: int + record_end: int + record_hex: str + protocol_view: str + + +def parse_structured_records(payload: bytes, record_type: RecordType) -> list[StructuredRecord]: + """Parse typed history blocks from raw or bit-shifted protocol payloads. + + Invalid candidates are ignored deliberately: callers use this parser only + as enrichment/fallback and retain the established decoder as their primary + path. + """ + marker = MONOPOLY_MARKER if record_type == "monopoly" else FORK_MARKER + for view_name, data in _iter_protocol_views(payload): + if marker not in data: + continue + records: list[StructuredRecord] = [] + search_from = 0 + while True: + marker_pos = data.find(marker, search_from) + if marker_pos < 0: + break + try: + parsed = _parse_block(data, marker_pos, record_type, marker, view_name) + except (StructuredProtocolError, UnicodeDecodeError): + parsed = [] + records.extend(parsed) + search_from = marker_pos + len(marker) + if records: + return records + return [] + + +def _parse_block( + data: bytes, + marker_pos: int, + record_type: RecordType, + marker: bytes, + view_name: str, +) -> list[StructuredRecord]: + pos = marker_pos + len(marker) + if _byte_at(data, pos) == 0: + pos += 1 + _reserved = _u32_at(data, pos) + declared_size = _u32_at(data, pos + 4) + row_count = _u32_at(data, pos + 8) + pos += 12 + if row_count > MAX_ROWS_PER_BLOCK: + raise StructuredProtocolError(f"row count is too large: {row_count}") + if declared_size > len(data) - pos: + raise StructuredProtocolError("declared block size exceeds payload") + + reader = _Reader(data, pos) + records = [] + for _row_index in range(row_count): + row_start = reader.pos + if record_type == "monopoly": + record = _parse_monopoly_row(reader, row_start, view_name) + else: + record = _parse_fork_row(reader, row_start, view_name) + records.append(record) + return records + + +def _parse_monopoly_row(reader: "_Reader", row_start: int, view_name: str) -> StructuredRecord: + roll_points_raw = reader.u32() + item_spec = reader.string() + _reserved = reader.u32() + secondary_count = reader.u32() + secondary_item_id = reader.string() + result_or_pool = reader.string() + + pool_pos = reader.pos + possible_pool = reader.try_string() + if possible_pool and possible_pool.startswith("CardPool_"): + pool_id = possible_pool + else: + reader.pos = pool_pos + pool_id = result_or_pool if result_or_pool.startswith("CardPool_") else None + + ticks = reader.u64() + return _make_record( + reader, + "monopoly", + item_spec, + ticks, + pool_id, + roll_points_raw, + secondary_item_id or None, + secondary_count, + row_start, + view_name, + ) + + +def _parse_fork_row(reader: "_Reader", row_start: int, view_name: str) -> StructuredRecord: + item_spec = reader.string() + pool_id = reader.string() + ticks = reader.u64() + return _make_record( + reader, + "fork", + item_spec, + ticks, + pool_id or None, + None, + None, + None, + row_start, + view_name, + ) + + +def _make_record( + reader: "_Reader", + record_type: RecordType, + item_spec: str, + ticks: int, + pool_id: str | None, + roll_points_raw: int | None, + secondary_item_id: str | None, + secondary_count: int | None, + row_start: int, + view_name: str, +) -> StructuredRecord: + item_id, count = _parse_item_spec(item_spec) + if not item_id: + raise StructuredProtocolError("structured item ID is empty") + timestamp_unix = (ticks - DOTNET_EPOCH_TICKS) / DOTNET_TICKS_PER_SECOND + if not MIN_UNIX_SECONDS <= timestamp_unix <= MAX_UNIX_SECONDS: + raise StructuredProtocolError("structured timestamp is out of range") + timestamp_decoded = datetime.fromtimestamp(timestamp_unix, timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + return StructuredRecord( + record_type=record_type, + item_id=item_id, + count=count, + ticks=ticks, + timestamp_unix=timestamp_unix, + timestamp_decoded=timestamp_decoded, + pool_id=pool_id, + roll_points_raw=roll_points_raw, + secondary_item_id=secondary_item_id, + secondary_count=secondary_count, + record_start=row_start, + record_end=reader.pos, + record_hex=reader.data[row_start : reader.pos].hex(), + protocol_view=view_name, + ) + + +def _parse_item_spec(value: str) -> tuple[str, int]: + item_id, separator, raw_count = value.rpartition(",") + if separator: + try: + count = int(raw_count) + except ValueError: + count = 0 + if item_id and count > 0: + return item_id, count + return value, 1 + + +class _Reader: + def __init__(self, data: bytes, pos: int) -> None: + self.data = data + self.pos = pos + + def u32(self) -> int: + value = _u32_at(self.data, self.pos) + self.pos += 4 + return value + + def u64(self) -> int: + value = _u64_at(self.data, self.pos) + self.pos += 8 + return value + + def string(self) -> str: + length_pos = self.pos + length = self.u32() + if length == 0 or length > MAX_STRING_LENGTH: + raise StructuredProtocolError(f"invalid string length {length} at {length_pos}") + end = self.pos + length + raw = self.data[self.pos:end] + if len(raw) != length: + raise StructuredProtocolError("string exceeds payload") + self.pos = end + if raw.endswith(b"\0"): + raw = raw[:-1] + return raw.decode("utf-8") + + def try_string(self) -> str | None: + start = self.pos + try: + return self.string() + except (StructuredProtocolError, UnicodeDecodeError): + self.pos = start + return None + + +def _iter_protocol_views(payload: bytes): + yield "raw", payload + for bit_shift in range(1, 8): + shifted = _decode_shifted_bytes(payload, byte_offset=8, bit_shift=bit_shift) + yield f"shift8:{bit_shift}", shifted + + +def _decode_shifted_bytes(data: bytes, *, byte_offset: int, bit_shift: int) -> bytes: + result = bytearray() + count = max(0, len(data) - byte_offset) + for index in range(count): + bit_pos = (byte_offset + index) * 8 + bit_shift + byte_pos, shift = divmod(bit_pos, 8) + if byte_pos >= len(data): + break + value = data[byte_pos] >> shift + if shift and byte_pos + 1 < len(data): + value |= data[byte_pos + 1] << (8 - shift) + result.append(value & 0xFF) + return bytes(result) + + +def _byte_at(data: bytes, pos: int) -> int: + try: + return data[pos] + except IndexError as exc: + raise StructuredProtocolError("byte exceeds payload") from exc + + +def _u32_at(data: bytes, pos: int) -> int: + try: + return struct.unpack_from(" int: + try: + return struct.unpack_from(" bytes: + raw = value.encode("utf-8") + b"\0" + return len(raw).to_bytes(4, "little") + raw + + +def monopoly_payload( + item_spec: str, + *, + ticks: int = STRUCTURED_TICKS, + roll_points: int = 2, + secondary_item_id: str = "", + secondary_count: int = 0, + pool_id: str = "CardPool_Character", +) -> bytes: + row = ( + roll_points.to_bytes(4, "little") + + fstring(item_spec) + + (0).to_bytes(4, "little") + + secondary_count.to_bytes(4, "little") + + fstring(secondary_item_id) + + fstring(item_spec.split(",", 1)[0]) + + fstring(pool_id) + + ticks.to_bytes(8, "little") + ) + return ( + MONOPOLY_MARKER + + b"\0" + + (0).to_bytes(4, "little") + + len(row).to_bytes(4, "little") + + (1).to_bytes(4, "little") + + row + ) + + +def fork_payload(item_spec: str, *, ticks: int = STRUCTURED_TICKS) -> bytes: + row = fstring(item_spec) + fstring("ForkLottery_AnHunQu") + ticks.to_bytes(8, "little") + return ( + FORK_MARKER + + b"\0" + + (0).to_bytes(4, "little") + + len(row).to_bytes(4, "little") + + (1).to_bytes(4, "little") + + row + ) + + +def bit_pack_after_eight_byte_header(payload: bytes, shift: int) -> bytes: + packed = int.from_bytes(payload, "little") << shift + return bytes(8) + packed.to_bytes(len(payload) + 1, "little") + + +class StructuredProtocolTests(unittest.TestCase): + def test_structured_monopoly_parser_enriches_the_existing_decoder(self): + payload = monopoly_payload( + "Fashion_vehicle_1010_V008,3", + secondary_item_id="Dice_ticket_02", + secondary_count=5, + ) + + rows = decode_response_records(payload) + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["decoder_mode"], "heuristic_enriched") + self.assertEqual(rows[0]["reward_id"], "Fashion_vehicle_1010_V008") + self.assertEqual(rows[0]["reward_name"], "Tiger Incoming! - Livery") + self.assertEqual(rows[0]["quantity"], 3) + self.assertEqual(rows[0]["dice"], 2) + self.assertEqual(rows[0]["secondary_reward_id"], "Dice_ticket_02") + self.assertEqual(rows[0]["secondary_quantity"], 5) + self.assertEqual(rows[0]["structured_pool_id"], "CardPool_Character") + + def test_structured_monopoly_parser_falls_back_when_heuristic_returns_no_rows(self): + payload = monopoly_payload("Dice_ticket_02,50", roll_points=0) + + with patch("nte_history_exporter.decoder.protocol._decode_aligned_response_records", return_value=[]): + rows = decode_response_records(payload) + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["decoder_mode"], "structured_fallback") + self.assertEqual(rows[0]["reward_id"], "Dice_ticket_02") + self.assertEqual(rows[0]["quantity"], 50) + self.assertEqual(rows[0]["result_type"], "points_gift") + + def test_structured_fork_parser_is_a_complete_fallback(self): + rows = parse_arc_response(fork_payload("fork_dustbin,2")) + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["decoder_mode"], "structured_fallback") + self.assertEqual(rows[0]["reward_id"], "fork_dustbin") + self.assertEqual(rows[0]["reward_name"], "Dangerous Game") + self.assertEqual(rows[0]["structured_pool_id"], "ForkLottery_AnHunQu") + + def test_structured_parser_realigns_bit_packed_payload(self): + payload = bit_pack_after_eight_byte_header(monopoly_payload("1003,1"), 3) + + rows = parse_structured_records(payload, "monopoly") + + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0].item_id, "1003") + self.assertEqual(rows[0].protocol_view, "shift8:3") + + def test_matching_structured_data_enriches_without_replacing_heuristic_identity(self): + heuristic_payload = fixture_payload("limited-points-gift-1") + original = decode_response_records(heuristic_payload)[0] + structured_ticks = original["timestamp_ticks"] // 4 + combined = heuristic_payload + monopoly_payload( + "1020,7", + ticks=structured_ticks, + roll_points=0, + pool_id="CardPool_Character", + ) + + enriched = decode_response_records(combined)[0] + + self.assertEqual(enriched["decoder_mode"], "heuristic_enriched") + self.assertEqual(enriched["quantity"], 7) + self.assertEqual(enriched["reward_id"], original["reward_id"]) + self.assertEqual(enriched["timestamp_raw_hex"], original["timestamp_raw_hex"]) + self.assertEqual(enriched["timestamp_ticks"], original["timestamp_ticks"]) + + def test_conflicting_structured_data_cannot_override_heuristic_record(self): + heuristic_payload = fixture_payload("limited-points-gift-1") + original = decode_response_records(heuristic_payload)[0] + combined = heuristic_payload + monopoly_payload( + "1003,99", + ticks=original["timestamp_ticks"] // 4, + roll_points=6, + ) + + decoded = decode_response_records(combined)[0] + + self.assertNotIn("decoder_mode", decoded) + self.assertEqual(decoded["reward_id"], original["reward_id"]) + self.assertEqual(decoded["quantity"], original["quantity"]) + self.assertEqual(decoded["dice"], original["dice"]) + + def test_malformed_structured_block_fails_closed(self): + malformed = ( + MONOPOLY_MARKER + + b"\0" + + (0).to_bytes(4, "little") + + (9999).to_bytes(4, "little") + + (1).to_bytes(4, "little") + ) + + self.assertEqual(decode_response_records(malformed), []) + + def test_structured_diagnostics_are_debug_only(self): + row = decode_response_records(monopoly_payload("1003,1"))[0] + row.update( + { + "uid": "stable-test-uid", + "export_record": True, + "pool_group_id": "Lottery_Permanent", + "timestamp_group_ordinal": 0, + } + ) + + export_record = build_export_json([row], [])["records"][0] + self.assertNotIn("decoder_mode", export_record) + self.assertNotIn("structured_pool_id", export_record) + self.assertNotIn("secondary_reward_id", export_record) + + with TemporaryDirectory() as tmp: + path = Path(tmp) / "debug.csv" + write_csv(path, [row]) + with path.open(newline="", encoding="utf-8") as handle: + debug_record = next(csv.DictReader(handle)) + self.assertEqual(debug_record["decoder_mode"], "heuristic_enriched") + self.assertEqual(debug_record["structured_pool_id"], "CardPool_Character") + + +if __name__ == "__main__": + unittest.main() From 73bb100cd60469e4d01abfb50c5ec0ea72616d1f Mon Sep 17 00:00:00 2001 From: Golumpa Date: Wed, 15 Jul 2026 19:48:11 +0100 Subject: [PATCH 4/8] Add safe snapshot and segment assembly Add envelope-aware snapshot and segment assembly for structured Monopoly and Arc history records. --- docs/limitations.md | 3 + docs/packet-format.md | 23 +- src/nte_history_exporter/decoder/arc.py | 56 +++- src/nte_history_exporter/decoder/protocol.py | 5 +- src/nte_history_exporter/decoder/run.py | 52 ++- .../decoder/structured_protocol.py | 295 +++++++++++++++++- src/nte_history_exporter/export/csv_export.py | 3 + tests/test_structured_protocol.py | 187 ++++++++++- 8 files changed, 601 insertions(+), 23 deletions(-) diff --git a/docs/limitations.md b/docs/limitations.md index 01c4a2b..beaff11 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -13,6 +13,9 @@ Known limitations: - Structured Monopoly and Arc blocks are parsed in enrichment/fallback mode. The existing decoder remains authoritative when structured rows do not agree on record count, reward ID, and timestamp. +- Structured snapshot/segment assembly is limited to runs where every decoded + row came from structured fallback. Existing primary-decoder runs are never + reordered. Ambiguous generations retain the last proven snapshot. - Live capture prefers Npcap on Windows and automatically falls back to the built-in raw-socket backend if Npcap is unavailable. Linux and macOS require the system libpcap runtime. - The file adapter reads mitmproxy `.flows` captures for research and testing. - Npcap is Windows-only and is not redistributed with this project; Linux and macOS use their system libpcap. diff --git a/docs/packet-format.md b/docs/packet-format.md index 50d7a74..ff5df85 100644 --- a/docs/packet-format.md +++ b/docs/packet-format.md @@ -21,8 +21,29 @@ The structured parser is deliberately compatibility-gated: - Malformed, incomplete, mismatched, or ambiguous structured data is ignored; it cannot overwrite a successfully decoded primary row. +Structured protocol envelopes identify a history stream, page, query side, and +segment index. For an all-structured fallback run, the snapshot assembler: + +- orders segments by their protocol index while retaining row order inside + every segment; +- ignores exact retransmissions; +- starts a new generation when an existing segment index changes; +- replaces an older snapshot only when the new generation covers at least the + same segment range; +- merges a partial generation only when its suffix has one unique overlap with + the proven snapshot; and +- retains the proven snapshot and records an assembly warning when a merge is + ambiguous. + +Assembly never runs on a history run containing a successfully decoded primary +row. Such runs retain their existing packet/page order exactly. Timestamp-group +ordinals and UIDs are calculated only after any fallback assembly, using the +same inputs and algorithms as before. + `decoder_mode`, `structured_protocol_view`, `structured_pool_id`, -`secondary_reward_id`, and `secondary_quantity` are research/debug CSV fields. +`structured_generation_index`, `structured_assembly`, +`structured_assembly_warning_count`, `secondary_reward_id`, and +`secondary_quantity` are research/debug CSV fields. They are intentionally omitted from the public JSON export, whose format stays at version 1. diff --git a/src/nte_history_exporter/decoder/arc.py b/src/nte_history_exporter/decoder/arc.py index 9f1bdc7..b42b1e8 100644 --- a/src/nte_history_exporter/decoder/arc.py +++ b/src/nte_history_exporter/decoder/arc.py @@ -22,7 +22,12 @@ from nte_history_exporter.constants import ( from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 from nte_history_exporter.decoder.run import fmt_packet_time from nte_history_exporter.mappings import ARC_META -from nte_history_exporter.decoder.structured_protocol import StructuredRecord, parse_structured_records +from nte_history_exporter.decoder.structured_protocol import ( + StructuredProtocolAssembler, + StructuredRecord, + parse_structured_blocks, + parse_structured_records, +) def is_arc_history_request(content: bytes) -> bool: @@ -125,7 +130,7 @@ def _arc_metadata(arc_id: str) -> dict[str, Any]: return next((meta for item_id, meta in ARC_META.items() if item_id.casefold() == folded), {}) -def _structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]: +def structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]: records = [] for structured in structured_rows: meta = _arc_metadata(structured.item_id) @@ -149,6 +154,7 @@ def _structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[s "decoder_mode": "structured_fallback", "structured_pool_id": structured.pool_id, "structured_protocol_view": structured.protocol_view, + "structured_generation_index": structured.generation_index, } ) return records @@ -176,10 +182,10 @@ def parse_arc_response(response: bytes) -> list[dict[str, Any]]: legacy_rows = _parse_legacy_arc_response(response) if legacy_rows: return _enrich_legacy_arc_rows(legacy_rows, structured_rows) - return _structured_arc_rows(structured_rows) + return structured_arc_rows(structured_rows) -def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: +def _build_primary_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: pool = POOL_META["arc_miracle_box"] rows: list[dict[str, Any]] = [] for pair in pairs: @@ -209,6 +215,48 @@ def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: return rows +def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: + primary_rows = _build_primary_arc_rows_from_pairs(pairs) + if primary_rows and any(row.get("decoder_mode") != "structured_fallback" for row in primary_rows): + return primary_rows + + assembler = StructuredProtocolAssembler() + for source_index, pair in enumerate(pairs): + assembler.add_blocks(parse_structured_blocks(pair[6], "fork", source_index=source_index)) + assembled = assembler.rows("fork") + if not assembled: + return primary_rows + + pool = POOL_META["arc_miracle_box"] + rows = [] + for row_index, (record, structured) in enumerate( + zip(structured_arc_rows(assembled), assembled), start=1 + ): + source_index = structured.source_index or 0 + pair = pairs[source_index] + page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7] + rows.append( + { + **record, + "page": page, + "offset": offset, + "row": row_index, + "pool_group_id": pool["id"], + "pool_group_name": pool["name"], + "request_msg": req_i, + "request_time_utc": fmt_packet_time(req_ts), + "response_msg": resp_i, + "response_time_utc": fmt_packet_time(resp_ts), + "response_len": len(response), + "record_count": len(assembled), + "structured_assembly": "snapshot_segments", + "structured_assembly_warning_count": len(assembler.warnings), + } + ) + annotate_arc_groups(rows) + return rows + + def make_arc_uid(timestamp_raw: str, ordinal: int) -> str: source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal)]) return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32] diff --git a/src/nte_history_exporter/decoder/protocol.py b/src/nte_history_exporter/decoder/protocol.py index c890257..3fed6c7 100644 --- a/src/nte_history_exporter/decoder/protocol.py +++ b/src/nte_history_exporter/decoder/protocol.py @@ -368,7 +368,7 @@ def _reward_metadata(reward_id: str) -> dict[str, Any]: return next((meta for item_id, meta in REWARDS_BY_ID.items() if item_id.casefold() == folded), {}) -def _structured_monopoly_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]: +def structured_monopoly_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]: rows = [] for row_index, structured in enumerate(structured_rows, start=1): dice, dice_raw, result_type, result_source = _structured_result(structured.roll_points_raw) @@ -405,6 +405,7 @@ def _structured_monopoly_rows(structured_rows: list[StructuredRecord]) -> list[d "secondary_reward_id": structured.secondary_item_id, "secondary_quantity": structured.secondary_count, "structured_protocol_view": structured.protocol_view, + "structured_generation_index": structured.generation_index, } ) return rows @@ -425,4 +426,4 @@ def decode_response_records(response_content: bytes) -> list[dict[str, Any]]: break if heuristic_rows: return _enrich_heuristic_rows(heuristic_rows, structured_rows) - return _structured_monopoly_rows(structured_rows) + return structured_monopoly_rows(structured_rows) diff --git a/src/nte_history_exporter/decoder/run.py b/src/nte_history_exporter/decoder/run.py index c756755..1b4a646 100644 --- a/src/nte_history_exporter/decoder/run.py +++ b/src/nte_history_exporter/decoder/run.py @@ -4,7 +4,11 @@ from datetime import datetime, timezone from typing import Any from nte_history_exporter.constants import POOL_META -from nte_history_exporter.decoder.protocol import decode_response_records +from nte_history_exporter.decoder.protocol import decode_response_records, structured_monopoly_rows +from nte_history_exporter.decoder.structured_protocol import ( + StructuredProtocolAssembler, + parse_structured_blocks, +) def fmt_packet_time(ts: float | None) -> str: @@ -13,7 +17,7 @@ def fmt_packet_time(ts: float | None) -> str: return datetime.fromtimestamp(ts, timezone.utc).strftime("%H:%M:%S.%f")[:-3] -def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: +def _build_primary_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: rows_out: list[dict[str, Any]] = [] for pair in pairs: page, offset, req_i, req_ts, resp_i, resp_ts, response_content = pair[:7] @@ -59,3 +63,47 @@ def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: } ) return rows_out + + +def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: + primary_rows = _build_primary_rows_from_pairs(pairs) + decoded_rows = [row for row in primary_rows if row.get("record_count", 0)] + if decoded_rows and any(row.get("decoder_mode") != "structured_fallback" for row in decoded_rows): + return primary_rows + + assembler = StructuredProtocolAssembler() + for source_index, pair in enumerate(pairs): + assembler.add_blocks( + parse_structured_blocks(pair[6], "monopoly", source_index=source_index) + ) + assembled = assembler.rows("monopoly") + if not assembled: + return primary_rows + + rows_out: list[dict[str, Any]] = [] + converted = structured_monopoly_rows(assembled) + for row_index, (record, structured) in enumerate(zip(converted, assembled), start=1): + source_index = structured.source_index or 0 + pair = pairs[source_index] + page, offset, req_i, req_ts, resp_i, resp_ts, response_content = pair[:7] + kind = pair[7] if len(pair) > 7 else "permanent" + pool = POOL_META.get(kind, POOL_META["permanent"]) + rows_out.append( + { + "page": page, + "offset": offset, + "pool_group_id": pool["id"], + "pool_group_name": pool["name"], + "request_msg": req_i, + "request_time_utc": fmt_packet_time(req_ts), + "response_msg": resp_i, + "response_time_utc": fmt_packet_time(resp_ts), + "response_len": len(response_content), + "record_count": len(assembled), + **record, + "row": row_index, + "structured_assembly": "snapshot_segments", + "structured_assembly_warning_count": len(assembler.warnings), + } + ) + return rows_out diff --git a/src/nte_history_exporter/decoder/structured_protocol.py b/src/nte_history_exporter/decoder/structured_protocol.py index c541147..e93f79e 100644 --- a/src/nte_history_exporter/decoder/structured_protocol.py +++ b/src/nte_history_exporter/decoder/structured_protocol.py @@ -1,7 +1,7 @@ from __future__ import annotations import struct -from dataclasses import dataclass +from dataclasses import dataclass, field, replace from datetime import datetime, timezone from typing import Literal @@ -16,6 +16,10 @@ DOTNET_EPOCH_TICKS = 621_355_968_000_000_000 DOTNET_TICKS_PER_SECOND = 10_000_000 MIN_UNIX_SECONDS = 1_500_000_000 MAX_UNIX_SECONDS = 4_102_444_800 +PROTOCOL_CONSTANT = 0x03000000 +MONOPOLY_BLOCK_KIND = 527 +FORK_BLOCK_KIND = 5906 +MONOPOLY_ENVELOPE_FOOTER = 1_774_080 class StructuredProtocolError(ValueError): @@ -38,6 +42,132 @@ class StructuredRecord: record_end: int record_hex: str protocol_view: str + source_index: int | None = None + generation_index: int | None = None + + +@dataclass(frozen=True) +class ProtocolEnvelope: + record_type: RecordType + stream_key: str + page_index: int + query_high: bool + segment_index: int + + +@dataclass(frozen=True) +class StructuredBlock: + record_type: RecordType + marker_offset: int + declared_size: int + rows: tuple[StructuredRecord, ...] + envelope: ProtocolEnvelope | None + + +@dataclass +class _Generation: + index: int + segments: dict[int, StructuredBlock] = field(default_factory=dict) + + +@dataclass +class _Stream: + generations: list[_Generation] = field(default_factory=list) + + +class StructuredProtocolAssembler: + """Assemble retransmitted and overlapping structured snapshot segments.""" + + def __init__(self) -> None: + self._stream_order: list[str] = [] + self._streams: dict[str, _Stream] = {} + self._legacy_rows: list[StructuredRecord] = [] + self.warnings: list[dict[str, str | int]] = [] + + def add_blocks(self, blocks: list[StructuredBlock]) -> None: + for block in blocks: + self.add_block(block) + + def add_block(self, block: StructuredBlock) -> bool: + envelope = block.envelope + if envelope is None: + if not self._legacy_rows: + self._stream_order.append("__legacy__") + self._legacy_rows.extend(block.rows) + return True + + stream = self._streams.get(envelope.stream_key) + if stream is None: + stream = _Stream() + self._streams[envelope.stream_key] = stream + self._stream_order.append(envelope.stream_key) + if not stream.generations: + stream.generations.append(_Generation(0)) + generation = stream.generations[-1] + existing = generation.segments.get(envelope.segment_index) + if existing is not None: + if _block_signature(existing) == _block_signature(block): + return False + generation = _Generation(len(stream.generations)) + stream.generations.append(generation) + generation.segments[envelope.segment_index] = block + return True + + def rows(self, record_type: RecordType | None = None) -> list[StructuredRecord]: + rows: list[StructuredRecord] = [] + for stream_key in self._stream_order: + if stream_key == "__legacy__": + rows.extend( + row + for row in self._legacy_rows + if record_type is None or row.record_type == record_type + ) + continue + assembled = self._assemble_stream(stream_key, self._streams[stream_key]) + rows.extend(row for row in assembled if record_type is None or row.record_type == record_type) + return rows + + def _assemble_stream(self, stream_key: str, stream: _Stream) -> list[StructuredRecord]: + result: list[StructuredRecord] = [] + result_max_segment: int | None = None + for generation in stream.generations: + if not generation.segments: + continue + generation_rows = _generation_rows(generation) + segment_indexes = sorted(generation.segments) + generation_min = segment_indexes[0] + generation_max = segment_indexes[-1] + if not result: + result = generation_rows + result_max_segment = generation_max + continue + if generation_min == 0: + if result_max_segment is None or generation_max >= result_max_segment: + result = generation_rows + result_max_segment = generation_max + continue + merged = _partial_snapshot_merge(generation_rows, result) + if merged is not None: + result = merged + else: + self._warn(stream_key, generation, "partial snapshot cannot be merged safely") + continue + if result_max_segment is not None and generation_min > result_max_segment: + result.extend(generation_rows) + result_max_segment = generation_max + continue + self._warn(stream_key, generation, "non-zero snapshot reset cannot be merged safely") + return result + + def _warn(self, stream_key: str, generation: _Generation, message: str) -> None: + warning = { + "code": "AMBIGUOUS_STRUCTURED_SNAPSHOT", + "stream_key": stream_key, + "generation_index": generation.index, + "message": message, + } + if warning not in self.warnings: + self.warnings.append(warning) def parse_structured_records(payload: bytes, record_type: RecordType) -> list[StructuredRecord]: @@ -47,24 +177,38 @@ def parse_structured_records(payload: bytes, record_type: RecordType) -> list[St as enrichment/fallback and retain the established decoder as their primary path. """ + assembler = StructuredProtocolAssembler() + assembler.add_blocks(parse_structured_blocks(payload, record_type)) + return assembler.rows(record_type) + + +def parse_structured_blocks( + payload: bytes, + record_type: RecordType, + *, + source_index: int | None = None, +) -> list[StructuredBlock]: marker = MONOPOLY_MARKER if record_type == "monopoly" else FORK_MARKER for view_name, data in _iter_protocol_views(payload): if marker not in data: continue - records: list[StructuredRecord] = [] + blocks: list[StructuredBlock] = [] search_from = 0 while True: marker_pos = data.find(marker, search_from) if marker_pos < 0: break try: - parsed = _parse_block(data, marker_pos, record_type, marker, view_name) + parsed = _parse_block( + data, marker_pos, record_type, marker, view_name, source_index + ) except (StructuredProtocolError, UnicodeDecodeError): - parsed = [] - records.extend(parsed) + parsed = None + if parsed is not None: + blocks.append(parsed) search_from = marker_pos + len(marker) - if records: - return records + if blocks: + return blocks return [] @@ -74,7 +218,9 @@ def _parse_block( record_type: RecordType, marker: bytes, view_name: str, -) -> list[StructuredRecord]: + source_index: int | None, +) -> StructuredBlock: + envelope = _parse_protocol_envelope(record_type, data, marker_pos, view_name) pos = marker_pos + len(marker) if _byte_at(data, pos) == 0: pos += 1 @@ -92,14 +238,22 @@ def _parse_block( for _row_index in range(row_count): row_start = reader.pos if record_type == "monopoly": - record = _parse_monopoly_row(reader, row_start, view_name) + record = _parse_monopoly_row(reader, row_start, view_name, source_index) else: - record = _parse_fork_row(reader, row_start, view_name) + record = _parse_fork_row(reader, row_start, view_name, source_index) records.append(record) - return records + return StructuredBlock( + record_type=record_type, + marker_offset=marker_pos, + declared_size=declared_size, + rows=tuple(records), + envelope=envelope, + ) -def _parse_monopoly_row(reader: "_Reader", row_start: int, view_name: str) -> StructuredRecord: +def _parse_monopoly_row( + reader: "_Reader", row_start: int, view_name: str, source_index: int | None +) -> StructuredRecord: roll_points_raw = reader.u32() item_spec = reader.string() _reserved = reader.u32() @@ -127,10 +281,13 @@ def _parse_monopoly_row(reader: "_Reader", row_start: int, view_name: str) -> St secondary_count, row_start, view_name, + source_index, ) -def _parse_fork_row(reader: "_Reader", row_start: int, view_name: str) -> StructuredRecord: +def _parse_fork_row( + reader: "_Reader", row_start: int, view_name: str, source_index: int | None +) -> StructuredRecord: item_spec = reader.string() pool_id = reader.string() ticks = reader.u64() @@ -145,6 +302,7 @@ def _parse_fork_row(reader: "_Reader", row_start: int, view_name: str) -> Struct None, row_start, view_name, + source_index, ) @@ -159,6 +317,7 @@ def _make_record( secondary_count: int | None, row_start: int, view_name: str, + source_index: int | None, ) -> StructuredRecord: item_id, count = _parse_item_spec(item_spec) if not item_id: @@ -182,6 +341,7 @@ def _make_record( record_end=reader.pos, record_hex=reader.data[row_start : reader.pos].hex(), protocol_view=view_name, + source_index=source_index, ) @@ -197,6 +357,108 @@ def _parse_item_spec(value: str) -> tuple[str, int]: return value, 1 +def _parse_protocol_envelope( + record_type: RecordType, + data: bytes, + marker_pos: int, + view_name: str, +) -> ProtocolEnvelope | None: + if marker_pos == 0 or not view_name.startswith("shift8:"): + return None + if record_type == "monopoly": + if marker_pos < 26: + raise StructuredProtocolError("monopoly envelope is truncated") + protocol_constant = _relative_u32(data, marker_pos, -26) + query_raw = _relative_u32(data, marker_pos, -22) + page_raw = _relative_u32(data, marker_pos, -18) + block_kind = _relative_u32(data, marker_pos, -14) + pool_token = _relative_u32(data, marker_pos, -10) + footer = _relative_u32(data, marker_pos, -6) + if ( + protocol_constant != PROTOCOL_CONSTANT + or block_kind != MONOPOLY_BLOCK_KIND + or footer != MONOPOLY_ENVELOPE_FOOTER + ): + raise StructuredProtocolError("invalid monopoly envelope constants") + stream_key = f"monopoly:{pool_token}" + else: + if marker_pos < 17: + raise StructuredProtocolError("fork envelope is truncated") + protocol_constant = _relative_u32(data, marker_pos, -17) + query_raw = _relative_u32(data, marker_pos, -13) + page_raw = _relative_u32(data, marker_pos, -9) + block_kind = _relative_u32(data, marker_pos, -5) + if protocol_constant != PROTOCOL_CONSTANT or block_kind != FORK_BLOCK_KIND: + raise StructuredProtocolError("invalid fork envelope constants") + stream_key = "fork" + page_index = page_raw & 0x7FFFFFFF + query_high = bool(query_raw & 0x80000000) + return ProtocolEnvelope( + record_type=record_type, + stream_key=stream_key, + page_index=page_index, + query_high=query_high, + segment_index=_segment_index(page_index, query_high), + ) + + +def _segment_index(page_index: int, query_high: bool) -> int: + if query_high: + return page_index * 2 + if page_index > 0: + return page_index * 2 - 1 + raise StructuredProtocolError("low query cannot describe page zero") + + +def _row_signature(row: StructuredRecord) -> tuple: + return ( + row.record_type, + row.ticks, + row.pool_id, + row.item_id, + row.count, + row.roll_points_raw, + row.secondary_item_id, + row.secondary_count, + ) + + +def _block_signature(block: StructuredBlock) -> tuple: + return block.record_type, tuple(_row_signature(row) for row in block.rows) + + +def _generation_rows(generation: _Generation) -> list[StructuredRecord]: + rows = [] + for segment_index in sorted(generation.segments): + block = generation.segments[segment_index] + rows.extend(replace(row, generation_index=generation.index) for row in block.rows) + return rows + + +def _partial_snapshot_merge( + new_rows: list[StructuredRecord], old_rows: list[StructuredRecord] +) -> list[StructuredRecord] | None: + if not new_rows: + return list(old_rows) + if not old_rows: + return list(new_rows) + new_signatures = [_row_signature(row) for row in new_rows] + old_signatures = [_row_signature(row) for row in old_rows] + max_overlap = min(len(new_signatures), len(old_signatures)) + matches: list[tuple[int, int]] = [] + for overlap in range(max_overlap, 0, -1): + suffix = new_signatures[-overlap:] + for position in range(len(old_signatures) - overlap + 1): + if old_signatures[position : position + overlap] == suffix: + matches.append((overlap, position)) + if matches: + break + if len(matches) != 1: + return None + overlap, position = matches[0] + return [*new_rows, *old_rows[position + overlap :]] + + class _Reader: def __init__(self, data: bytes, pos: int) -> None: self.data = data @@ -271,6 +533,13 @@ def _u32_at(data: bytes, pos: int) -> int: raise StructuredProtocolError("u32 exceeds payload") from exc +def _relative_u32(data: bytes, marker_pos: int, relative_pos: int) -> int: + pos = marker_pos + relative_pos + if pos < 0: + raise StructuredProtocolError("envelope position precedes payload") + return _u32_at(data, pos) + + def _u64_at(data: bytes, pos: int) -> int: try: return struct.unpack_from(" bytes: return bytes(8) + packed.to_bytes(len(payload) + 1, "little") +def enveloped_monopoly_payload( + item_spec: str, + *, + page_index: int, + query_high: bool, + pool_token: int = 256, + shift: int = 3, +) -> bytes: + envelope = ( + PROTOCOL_CONSTANT.to_bytes(4, "little") + + ((0x80000000 if query_high else 0)).to_bytes(4, "little") + + page_index.to_bytes(4, "little") + + MONOPOLY_BLOCK_KIND.to_bytes(4, "little") + + pool_token.to_bytes(4, "little") + + MONOPOLY_ENVELOPE_FOOTER.to_bytes(4, "little") + + b"\0\0" + ) + return bit_pack_after_eight_byte_header(envelope + monopoly_payload(item_spec), shift) + + +def enveloped_fork_payload( + item_spec: str, + *, + page_index: int, + query_high: bool, + shift: int = 3, +) -> bytes: + envelope = ( + PROTOCOL_CONSTANT.to_bytes(4, "little") + + ((0x80000000 if query_high else 0)).to_bytes(4, "little") + + page_index.to_bytes(4, "little") + + FORK_BLOCK_KIND.to_bytes(4, "little") + + b"\0" + ) + return bit_pack_after_eight_byte_header(envelope + fork_payload(item_spec), shift) + + +def assembled_block(item_id: str, segment_index: int, source_index: int = 0): + block = parse_structured_blocks( + monopoly_payload(item_id), "monopoly", source_index=source_index + )[0] + envelope = ProtocolEnvelope( + record_type="monopoly", + stream_key="monopoly:256", + page_index=segment_index // 2, + query_high=segment_index % 2 == 0, + segment_index=segment_index, + ) + return replace(block, envelope=envelope) + + class StructuredProtocolTests(unittest.TestCase): def test_structured_monopoly_parser_enriches_the_existing_decoder(self): payload = monopoly_payload( @@ -122,6 +187,126 @@ class StructuredProtocolTests(unittest.TestCase): self.assertEqual(rows[0].item_id, "1003") self.assertEqual(rows[0].protocol_view, "shift8:3") + def test_protocol_envelope_exposes_stream_and_segment_identity(self): + payload = enveloped_monopoly_payload("1003", page_index=2, query_high=False) + + block = parse_structured_blocks(payload, "monopoly")[0] + + self.assertIsNotNone(block.envelope) + self.assertEqual(block.envelope.stream_key, "monopoly:256") + self.assertEqual(block.envelope.page_index, 2) + self.assertFalse(block.envelope.query_high) + self.assertEqual(block.envelope.segment_index, 3) + + def test_assembler_orders_segments_and_ignores_retransmissions(self): + assembler = StructuredProtocolAssembler() + segment_one = assembled_block("1010", 1) + assembler.add_blocks([segment_one, assembled_block("1003", 0), segment_one]) + + rows = assembler.rows("monopoly") + + self.assertEqual([row.item_id for row in rows], ["1003", "1010"]) + self.assertEqual([row.generation_index for row in rows], [0, 0]) + self.assertEqual(assembler.warnings, []) + + def test_assembler_never_deduplicates_blocks_without_envelopes(self): + assembler = StructuredProtocolAssembler() + block = parse_structured_blocks(monopoly_payload("1003"), "monopoly")[0] + + assembler.add_blocks([block, block]) + + self.assertEqual([row.item_id for row in assembler.rows("monopoly")], ["1003", "1003"]) + + def test_new_complete_generation_replaces_old_snapshot(self): + assembler = StructuredProtocolAssembler() + assembler.add_blocks([assembled_block("1003", 0), assembled_block("1010", 1)]) + assembler.add_blocks([assembled_block("1020", 0), assembled_block("1021", 1)]) + + rows = assembler.rows("monopoly") + + self.assertEqual([row.item_id for row in rows], ["1020", "1021"]) + self.assertEqual([row.generation_index for row in rows], [1, 1]) + + def test_partial_generation_merges_only_on_unique_overlap(self): + assembler = StructuredProtocolAssembler() + assembler.add_blocks( + [ + assembled_block("1003", 0), + assembled_block("1010", 1), + assembled_block("1020", 2), + ] + ) + assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)]) + + rows = assembler.rows("monopoly") + + self.assertEqual([row.item_id for row in rows], ["1099", "1010", "1020"]) + self.assertEqual(assembler.warnings, []) + + def test_ambiguous_partial_generation_keeps_proven_snapshot(self): + assembler = StructuredProtocolAssembler() + assembler.add_blocks( + [ + assembled_block("1003", 0), + assembled_block("1010", 1), + assembled_block("1003", 2), + assembled_block("1010", 3), + ] + ) + assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)]) + + rows = assembler.rows("monopoly") + + self.assertEqual([row.item_id for row in rows], ["1003", "1010", "1003", "1010"]) + self.assertEqual(assembler.warnings[0]["code"], "AMBIGUOUS_STRUCTURED_SNAPSHOT") + + def test_pair_assembly_is_used_only_for_all_structured_fallback(self): + segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False) + segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True) + pairs = [ + (2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"), + (1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"), + ] + + with patch("nte_history_exporter.decoder.protocol._decode_aligned_response_records", return_value=[]): + rows = build_rows_from_pairs(pairs) + annotated = annotate_groups(rows) + + self.assertEqual([row["reward_id"] for row in annotated], ["1003", "1010"]) + self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in annotated)) + self.assertEqual(annotated[0]["uid"], make_uid(annotated[0], 0)) + + def test_pair_assembly_cannot_reorder_successful_heuristic_rows(self): + segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False) + segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True) + pairs = [ + (2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"), + (1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"), + ] + + rows = build_rows_from_pairs(pairs) + + self.assertEqual([row["reward_id"] for row in rows], ["1010", "1003"]) + self.assertTrue(all(row["decoder_mode"] == "heuristic_enriched" for row in rows)) + self.assertTrue(all("structured_assembly" not in row for row in rows)) + + def test_arc_fallback_uses_the_same_segment_assembly_and_uid_order(self): + segment_one = enveloped_fork_payload("fork_vine", page_index=1, query_high=False) + segment_zero = enveloped_fork_payload("fork_dustbin", page_index=0, query_high=True) + pairs = [ + (2, 4, 1, 1.0, 2, 1.1, segment_one, "arc_miracle_box"), + (1, 2, 3, 1.2, 4, 1.3, segment_zero, "arc_miracle_box"), + ] + + rows = build_arc_rows_from_pairs(pairs) + + self.assertEqual([row["reward_id"] for row in rows], ["fork_dustbin", "fork_vine"]) + self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in rows)) + self.assertEqual( + rows[0]["uid"], + make_arc_uid(rows[0]["timestamp_raw_hex"], rows[0]["timestamp_group_ordinal"]), + ) + def test_matching_structured_data_enriches_without_replacing_heuristic_identity(self): heuristic_payload = fixture_payload("limited-points-gift-1") original = decode_response_records(heuristic_payload)[0] From 2b33ea63af287a87e4507193536d362f3467e829 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Thu, 16 Jul 2026 00:12:02 +0100 Subject: [PATCH 5/8] Add capture diagnostics and replay fixtures Add privacy-safe capture diagnostics and sanitized replay fixtures without changing the existing export format, record order, or UID generation. --- README.md | 7 +- docs/capture-diagnostics.md | 27 +++++ .../adapters/mitmproxy_flows.py | 1 + src/nte_history_exporter/cli.py | 11 +- .../live_capture/diagnostics.py | 90 +++++++++++++++ .../live_capture/runner.py | 14 ++- .../live_capture/session.py | 106 +++++++++++++++-- tests/fixtures/README.md | 6 + .../synthetic_capture_diagnostics.json | 60 ++++++++++ tests/support.py | 28 +++++ tests/test_capture_diagnostics.py | 108 ++++++++++++++++++ 11 files changed, 445 insertions(+), 13 deletions(-) create mode 100644 docs/capture-diagnostics.md create mode 100644 src/nte_history_exporter/live_capture/diagnostics.py create mode 100644 tests/fixtures/synthetic_capture_diagnostics.json create mode 100644 tests/test_capture_diagnostics.py diff --git a/README.md b/README.md index 2ffd974..a30c397 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ Decodes a `mitmproxy .flows` capture instead of listening live — used for rese | Flag | Effect | | --------- | --------------------------------------------------- | | `--live` | Capture live UDP traffic instead of reading a file. | -| `--debug` | Also write the full research CSV next to each JSON. | +| `--debug` | Also write the research CSV and privacy-safe capture diagnostics. | | `--user-uid ` | Override the auto-detected NTE user UID in the JSON export. | | `--copy-clipboard` | Copy a single live export JSON to clipboard after saving. | @@ -156,7 +156,10 @@ Advanced live-capture selection: --capture-backend raw Require the Windows raw-socket backend ``` -The `--debug` CSV holds any extra information that might be needed for fixing bugs. It contains no dangerous personal account data — only the raw bytes of the captured history page. +The `--debug` CSV holds decoded research fields, including raw captured history +records. A separate versioned `*.diagnostics.json` sidecar provides shareable +reason codes and counts without payloads, network addresses, ports, packet +timestamps, or user UID values. See [Capture diagnostics](docs/capture-diagnostics.md). The exporter automatically includes the shareable NTE user UID when it appears in the capture. If a short capture does not include it, the console asks before saving; you can also pass it explicitly with `--user-uid`. diff --git a/docs/capture-diagnostics.md b/docs/capture-diagnostics.md new file mode 100644 index 0000000..9f92b0d --- /dev/null +++ b/docs/capture-diagnostics.md @@ -0,0 +1,27 @@ +# Capture diagnostics + +Running the exporter with `--debug` writes a UID-free `Capture_*.diagnostics.json` +file in the export directory. This sidecar explains what the capture +pipeline recognized, rejected, and paired without changing the public export +format or record order. It is also written when no history page could be +exported. + +The report contains bounded events, aggregate counters, reason codes, packet +positions, payload lengths, history kinds, page numbers, and record counts. It +does not contain packet payloads, IP addresses, ports, packet timestamps, or a +user UID. The report format is versioned independently as +`nte-capture-diagnostics` version 1. + +Useful rejection codes include: + +- `RESPONSE_TOO_SHORT`: a matching inbound packet could not contain a history + response; +- `NO_HISTORY_MARKER`: a matching response candidate had no known history + marker; +- `HISTORY_MARKER_PARSE_FAILED`: a known marker was present but neither decoder + produced records; +- `RESPONSE_KIND_MISMATCH`: decoded data did not match the pending history kind; +- `REQUEST_REPLACED`: a recovery request superseded an unanswered request. + +Events are capped at 200 per session. Aggregate counters continue after that +limit and `events_omitted` records how many event entries were left out. diff --git a/src/nte_history_exporter/adapters/mitmproxy_flows.py b/src/nte_history_exporter/adapters/mitmproxy_flows.py index 19c35f7..fc2b930 100644 --- a/src/nte_history_exporter/adapters/mitmproxy_flows.py +++ b/src/nte_history_exporter/adapters/mitmproxy_flows.py @@ -119,4 +119,5 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d "arc_rows": arc_rows, "arc_warnings": arc_warnings, "user_uid": session.user_uid or user_uid, + "capture_diagnostics": session.diagnostic_report(), } diff --git a/src/nte_history_exporter/cli.py b/src/nte_history_exporter/cli.py index 09b7368..5b0090e 100644 --- a/src/nte_history_exporter/cli.py +++ b/src/nte_history_exporter/cli.py @@ -10,6 +10,7 @@ from nte_history_exporter.decoder.boundary import annotate_groups from nte_history_exporter.export.csv_export import write_csv from nte_history_exporter.export.json_export import build_export_json from nte_history_exporter.live_capture.libpcap import LibpcapUnavailable +from nte_history_exporter.live_capture.diagnostics import new_diagnostics_path, write_capture_diagnostics from nte_history_exporter.live_capture.runner import export_paths, run_live_capture from nte_history_exporter.update_check import check_for_update @@ -32,7 +33,11 @@ def build_parser() -> argparse.ArgumentParser: ), ) parser.add_argument("--copy-clipboard", action="store_true", help="copy a single live export to clipboard") - parser.add_argument("--debug", action="store_true", help="also write research CSVs next to the JSON exports") + parser.add_argument( + "--debug", + action="store_true", + help="also write a research CSV and privacy-safe capture diagnostics", + ) parser.add_argument("--user-uid", default=None, help="override the auto-detected NTE user UID in the JSON export") return parser @@ -79,8 +84,11 @@ def main(argv: list[str] | None = None) -> int: resolved_user_uid = console.prompt_user_uid() out_path, json_path = export_paths(kind, resolved_user_uid) + diagnostics_path = None if args.debug: write_csv(out_path, rows) + diagnostics_path = new_diagnostics_path(out_path.parent) + write_capture_diagnostics(diagnostics_path, decoded["capture_diagnostics"]) export = build_export_json( rows, warnings, @@ -105,6 +113,7 @@ def main(argv: list[str] | None = None) -> int: print() if args.debug: console.print_note(f"CSV written: {out_path}") + console.print_note(f"Diagnostics written: {diagnostics_path}") console.print_note(f"Export written: {json_path}") return 0 diff --git a/src/nte_history_exporter/live_capture/diagnostics.py b/src/nte_history_exporter/live_capture/diagnostics.py new file mode 100644 index 0000000..6f9ac6c --- /dev/null +++ b/src/nte_history_exporter/live_capture/diagnostics.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import json +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Iterable + + +DIAGNOSTIC_FORMAT = "nte-capture-diagnostics" +DIAGNOSTIC_FORMAT_VERSION = 1 +MAX_EVENTS = 200 + + +class CaptureDiagnostics: + """Collect bounded, privacy-safe observations about a capture session.""" + + def __init__(self) -> None: + self.counters: Counter[str] = Counter() + self.event_counts: Counter[str] = Counter() + self.reason_counts: Counter[str] = Counter() + self.events: list[dict[str, Any]] = [] + + def observe_packet(self, protocol: str) -> None: + self.counters["packets_seen"] += 1 + if protocol == "udp": + self.counters["udp_packets_seen"] += 1 + else: + self.counters["non_udp_packets_ignored"] += 1 + + def add_event( + self, + code: str, + packet_index: int, + *, + reason: bool = False, + **fields: Any, + ) -> None: + self.event_counts[code] += 1 + if reason: + self.reason_counts[code] += 1 + if len(self.events) >= MAX_EVENTS: + self.counters["events_omitted"] += 1 + return + event: dict[str, Any] = {"packet_index": packet_index, "code": code} + event.update({key: value for key, value in fields.items() if value is not None}) + self.events.append(event) + + def report(self, pending_requests: Iterable[Any]) -> dict[str, Any]: + pending = [ + { + "kind": request.kind, + "page": request.page, + "response_candidates": request.response_candidates, + "response_candidate_lengths": list(request.response_candidate_lengths), + } + for request in pending_requests + ] + return { + "format": DIAGNOSTIC_FORMAT, + "format_version": DIAGNOSTIC_FORMAT_VERSION, + "privacy": { + "contains_network_addresses": False, + "contains_network_ports": False, + "contains_packet_timestamps": False, + "contains_payload_bytes": False, + "contains_user_uid": False, + }, + "counters": dict(sorted(self.counters.items())), + "event_counts": dict(sorted(self.event_counts.items())), + "reason_counts": dict(sorted(self.reason_counts.items())), + "events": list(self.events), + "pending_requests": pending, + } + + +def new_diagnostics_path(output_dir: str | Path = "exports") -> Path: + directory = Path(output_dir) + directory.mkdir(parents=True, exist_ok=True) + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + path = directory / f"Capture_{stamp}.diagnostics.json" + counter = 2 + while path.exists(): + path = directory / f"Capture_{stamp}_{counter}.diagnostics.json" + counter += 1 + return path + + +def write_capture_diagnostics(path: str | Path, report: dict[str, Any]) -> None: + Path(path).write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/src/nte_history_exporter/live_capture/runner.py b/src/nte_history_exporter/live_capture/runner.py index 3f41f50..6c16d79 100644 --- a/src/nte_history_exporter/live_capture/runner.py +++ b/src/nte_history_exporter/live_capture/runner.py @@ -13,6 +13,7 @@ from nte_history_exporter.decoder.boundary import annotate_groups, select_contin from nte_history_exporter.export.csv_export import write_csv from nte_history_exporter.export.json_export import build_export_json from nte_history_exporter.live_capture.backends import open_capture_backend +from nte_history_exporter.live_capture.diagnostics import new_diagnostics_path, write_capture_diagnostics from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket from nte_history_exporter.live_capture.stop_key import StopKeyMonitor from nte_history_exporter.live_capture.windows_raw import detect_local_ipv4 @@ -135,6 +136,10 @@ def run_live_capture( ) exports = [] + diagnostics_path = None + if write_debug_csv: + diagnostics_path = new_diagnostics_path() + write_capture_diagnostics(diagnostics_path, session.diagnostic_report()) resolved_user_uid = user_uid or session.user_uid if session.kinds_seen() and not resolved_user_uid: resolved_user_uid = console.prompt_user_uid() @@ -164,6 +169,7 @@ def run_live_capture( { "kind": kind, "csv_path": csv_path if write_debug_csv else None, + "diagnostics_path": diagnostics_path if write_debug_csv else None, "json_path": json_path, "export": export, "payload": payload, @@ -176,7 +182,9 @@ def run_live_capture( console.print_note("Make sure the capture backend is running, then reopen the") console.print_note("history screen and scroll from page 1. If no page messages") console.print_note("appear, return to the main menu and re-enter the game.") - return {"exports": []} + if diagnostics_path is not None: + console.print_note(f"Diagnostics written: {diagnostics_path}") + return {"exports": [], "diagnostics_path": diagnostics_path} for item in exports: scan = item["export"]["scan"] @@ -194,6 +202,8 @@ def run_live_capture( if item["csv_path"] is not None: console.print_note(f"CSV written: {item['csv_path']}") console.print_note(f"Export written: {item['json_path']}") + if diagnostics_path is not None: + console.print_note(f"Diagnostics written: {diagnostics_path}") if copy_clipboard and len(exports) == 1: if copy_to_clipboard(exports[0]["payload"]): @@ -203,7 +213,7 @@ def run_live_capture( elif copy_clipboard and len(exports) > 1: console.print_note("Multiple banners captured; clipboard copy skipped so one export") console.print_note("does not overwrite another.") - return {"exports": exports} + return {"exports": exports, "diagnostics_path": diagnostics_path} def export_paths(kind: str, user_uid: str | None = None) -> tuple[Path, Path]: diff --git a/src/nte_history_exporter/live_capture/session.py b/src/nte_history_exporter/live_capture/session.py index e793294..171e782 100644 --- a/src/nte_history_exporter/live_capture/session.py +++ b/src/nte_history_exporter/live_capture/session.py @@ -17,9 +17,12 @@ from nte_history_exporter.decoder.protocol import ( history_request_kind, is_history_request, request_page, + response_contains_history_marker, ) from nte_history_exporter.decoder.run import build_rows_from_pairs +from nte_history_exporter.decoder.structured_protocol import FORK_MARKER, MONOPOLY_MARKER from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates +from nte_history_exporter.live_capture.diagnostics import CaptureDiagnostics @dataclass @@ -61,6 +64,7 @@ class LiveHistorySession: self.unanswered_pages: dict[str, dict[int, str]] = {} self.user_uid: str | None = None self.user_uid_candidates: Counter[str] = Counter() + self.diagnostics = CaptureDiagnostics() def _mark_unanswered(self, request: PendingRequest) -> None: if request.response_candidates: @@ -89,6 +93,14 @@ class LiveHistorySession: ) if same_stream and (request.page == 1 or pending.page == request.page): self._mark_unanswered(pending) + self.diagnostics.add_event( + "REQUEST_REPLACED", + request.request_msg, + reason=True, + kind=pending.kind, + page=pending.page, + response_candidates=pending.response_candidates, + ) else: retained.append(pending) self.pending = retained @@ -97,6 +109,7 @@ class LiveHistorySession: def process_packet(self, packet: UdpPacket) -> bool: self.packet_count += 1 + self.diagnostics.observe_packet(packet.protocol) candidates = extract_user_uid_candidates(packet.payload) if candidates: self.user_uid_candidates.update(candidates) @@ -118,6 +131,13 @@ class LiveHistorySession: dst_port=packet.dst_port, ) self._queue_request(req) + self.diagnostics.counters["history_requests_recognized"] += 1 + self.diagnostics.add_event( + "HISTORY_REQUEST_RECOGNIZED", + self.packet_count, + kind=req.kind, + page=req.page, + ) self.last_page_seen = req.page return False @@ -135,10 +155,17 @@ class LiveHistorySession: dst_port=packet.dst_port, ) self._queue_request(req) + self.diagnostics.counters["history_requests_recognized"] += 1 + self.diagnostics.add_event( + "HISTORY_REQUEST_RECOGNIZED", + self.packet_count, + kind=req.kind, + page=req.page, + ) self.last_page_seen = req.page return False - if packet.dst_ip != self.local_ip or len(packet.payload) < 100: + if packet.dst_ip != self.local_ip: return False connection_candidates = [ @@ -154,6 +181,14 @@ class LiveHistorySession: if not connection_candidates: return False + if len(packet.payload) < 100: + self._record_rejected_candidate( + connection_candidates, + packet.payload, + "RESPONSE_TOO_SHORT", + ) + return False + monopoly_records = decode_response_records(packet.payload) arc_records = parse_arc_response(packet.payload) if not monopoly_records else [] if monopoly_records: @@ -166,15 +201,36 @@ class LiveHistorySession: candidates = connection_candidates records = [] - if not records or not candidates: - for req in candidates: - req.response_candidates += 1 - req.response_candidate_lengths = ( - *req.response_candidate_lengths[-4:], - len(packet.payload), - ) + if records and not candidates: + self._record_rejected_candidate( + connection_candidates, + packet.payload, + "RESPONSE_KIND_MISMATCH", + ) return False + if not records: + reason = ( + "HISTORY_MARKER_PARSE_FAILED" + if response_contains_history_marker(packet.payload) + or MONOPOLY_MARKER in packet.payload + or FORK_MARKER in packet.payload + else "NO_HISTORY_MARKER" + ) + self._record_rejected_candidate(candidates, packet.payload, reason) + return False + + decoder_modes = sorted({record.get("decoder_mode", "heuristic") for record in records}) + self.diagnostics.counters["history_responses_decoded"] += 1 + self.diagnostics.add_event( + "HISTORY_RESPONSE_DECODED", + self.packet_count, + kind=candidates[0].kind, + payload_length=len(packet.payload), + record_count=len(records), + decoder_modes=decoder_modes, + ) + page_count = max(1, (len(records) + 4) // 5) if len(records) < 5: selected = [candidates[-1]] @@ -208,9 +264,43 @@ class LiveHistorySession: ) self.last_match_time = packet.timestamp self.last_page_seen = req.page + self.diagnostics.counters["pages_matched"] += 1 + self.diagnostics.add_event( + "PAGE_RESPONSE_MATCHED", + self.packet_count, + kind=req.kind, + page=req.page, + record_count=slice_count, + ) return bool(selected) + def _record_rejected_candidate( + self, + requests: list[PendingRequest], + payload: bytes, + reason: str, + ) -> None: + for req in requests: + req.response_candidates += 1 + req.response_candidate_lengths = ( + *req.response_candidate_lengths[-4:], + len(payload), + ) + self.diagnostics.counters["response_candidates_rejected"] += 1 + self.diagnostics.add_event( + reason, + self.packet_count, + reason=True, + kind=requests[0].kind if requests else None, + page=requests[0].page if requests else None, + payload_length=len(payload), + matching_requests=len(requests), + ) + + def diagnostic_report(self) -> dict[str, Any]: + return self.diagnostics.report(self.pending) + def kinds_seen(self) -> list[str]: seen = [] for pair in self.pairs: diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index abdedc9..2bec598 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -22,6 +22,12 @@ Do not replace this file with a real `.pcap`, `.flows`, or exported account history. Add new cases by constructing the smallest relevant payload, replacing all timestamps and endpoints, and extending the privacy assertions. +`synthetic_capture_diagnostics.json` is a smaller replay transcript for failure +paths. It deliberately contains a short response and a marker-free response so +the debug sidecar's reason codes and privacy contract can be tested without a +real capture. Repeated-byte payloads use `payload_byte` plus `payload_length` to +keep the fixture readable. + ## Synthetic NTE_Assets fixture `nte_assets/` mirrors only the six table paths and English localization file diff --git a/tests/fixtures/synthetic_capture_diagnostics.json b/tests/fixtures/synthetic_capture_diagnostics.json new file mode 100644 index 0000000..3766e4d --- /dev/null +++ b/tests/fixtures/synthetic_capture_diagnostics.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "description": "Minimal synthetic replay for capture diagnostic reason codes.", + "privacy": { + "synthetic": true, + "contains_user_uid": false, + "contains_raw_account_session": false + }, + "local_ip": "192.0.2.10", + "expected": { + "packets_seen": 3, + "history_requests_recognized": 1, + "response_candidates_rejected": 2, + "event_counts": { + "HISTORY_REQUEST_RECOGNIZED": 1, + "NO_HISTORY_MARKER": 1, + "RESPONSE_TOO_SHORT": 1 + }, + "reason_counts": { + "NO_HISTORY_MARKER": 1, + "RESPONSE_TOO_SHORT": 1 + }, + "pending_response_candidates": 2, + "pending_candidate_lengths": [80, 220] + }, + "packets": [ + { + "label": "permanent-page-1-request", + "timestamp": 1893456000.0, + "src_ip": "192.0.2.10", + "dst_ip": "198.51.100.20", + "src_port": 50000, + "dst_port": 40000, + "protocol": "udp", + "payload_hex": "00000000000000000000000000000000000000000000000000000000000000040000007c100000000400000000" + }, + { + "label": "matching-response-too-short", + "timestamp": 1893456000.1, + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 40000, + "dst_port": 50000, + "protocol": "udp", + "payload_byte": "00", + "payload_length": 80 + }, + { + "label": "matching-response-without-history-marker", + "timestamp": 1893456000.2, + "src_ip": "198.51.100.20", + "dst_ip": "192.0.2.10", + "src_port": 40000, + "dst_port": 50000, + "protocol": "udp", + "payload_byte": "00", + "payload_length": 220 + } + ] +} diff --git a/tests/support.py b/tests/support.py index d64467d..58a9a55 100644 --- a/tests/support.py +++ b/tests/support.py @@ -60,6 +60,34 @@ def load_network_fixture(): return json.load(f) +def load_capture_diagnostics_fixture(): + path = FIXTURES / "synthetic_capture_diagnostics.json" + with path.open(encoding="utf-8") as f: + return json.load(f) + + +def diagnostic_fixture_session(): + fixture = load_capture_diagnostics_fixture() + session = LiveHistorySession(fixture["local_ip"]) + for packet in fixture["packets"]: + if "payload_hex" in packet: + payload = bytes.fromhex(packet["payload_hex"]) + else: + payload = bytes.fromhex(packet["payload_byte"]) * packet["payload_length"] + session.process_packet( + UdpPacket( + timestamp=packet["timestamp"], + src_ip=packet["src_ip"], + dst_ip=packet["dst_ip"], + src_port=packet["src_port"], + dst_port=packet["dst_port"], + payload=payload, + protocol=packet["protocol"], + ) + ) + return session + + def fixture_packets(scenario=None): fixture = load_network_fixture() packets = fixture["packets"] diff --git a/tests/test_capture_diagnostics.py b/tests/test_capture_diagnostics.py new file mode 100644 index 0000000..9a2fbdd --- /dev/null +++ b/tests/test_capture_diagnostics.py @@ -0,0 +1,108 @@ +from tests.support import * # noqa: F401,F403 + +from nte_history_exporter.live_capture.diagnostics import ( + new_diagnostics_path, + write_capture_diagnostics, +) + + +class CaptureDiagnosticsTests(unittest.TestCase): + def test_successful_network_replay_reports_capture_pipeline_counts(self): + report = fixture_session().diagnostic_report() + + self.assertEqual( + report["counters"], + { + "history_requests_recognized": 10, + "history_responses_decoded": 10, + "packets_seen": 20, + "pages_matched": 10, + "udp_packets_seen": 20, + }, + ) + self.assertEqual(report["reason_counts"], {}) + self.assertEqual(report["pending_requests"], []) + + def test_synthetic_replay_reports_actionable_rejection_reasons(self): + fixture = load_capture_diagnostics_fixture() + report = diagnostic_fixture_session().diagnostic_report() + expected = fixture["expected"] + + self.assertEqual(report["format"], "nte-capture-diagnostics") + self.assertEqual(report["format_version"], 1) + self.assertEqual(report["counters"]["packets_seen"], expected["packets_seen"]) + self.assertEqual( + report["counters"]["history_requests_recognized"], + expected["history_requests_recognized"], + ) + self.assertEqual( + report["counters"]["response_candidates_rejected"], + expected["response_candidates_rejected"], + ) + self.assertEqual(report["event_counts"], expected["event_counts"]) + self.assertEqual(report["reason_counts"], expected["reason_counts"]) + self.assertEqual(report["pending_requests"][0]["response_candidates"], 2) + self.assertEqual( + report["pending_requests"][0]["response_candidate_lengths"], + expected["pending_candidate_lengths"], + ) + + def test_diagnostic_fixture_contains_only_synthetic_network_identity(self): + fixture = load_capture_diagnostics_fixture() + self.assertTrue(fixture["privacy"]["synthetic"]) + self.assertFalse(fixture["privacy"]["contains_user_uid"]) + self.assertFalse(fixture["privacy"]["contains_raw_account_session"]) + + allowed_ips = {"192.0.2.10", "198.51.100.20"} + for packet in fixture["packets"]: + with self.subTest(label=packet["label"]): + self.assertIn(packet["src_ip"], allowed_ips) + self.assertIn(packet["dst_ip"], allowed_ips) + if "payload_hex" in packet: + payload = bytes.fromhex(packet["payload_hex"]) + else: + payload = bytes.fromhex(packet["payload_byte"]) * packet["payload_length"] + self.assertIsNone(extract_user_uid(payload)) + + def test_diagnostic_report_excludes_capture_identity_and_payload_data(self): + report = diagnostic_fixture_session().diagnostic_report() + serialized = json.dumps(report) + forbidden_keys = { + "src_ip", + "dst_ip", + "src_port", + "dst_port", + "timestamp", + "payload", + "payload_hex", + "user_uid", + } + + def assert_safe(value): + if isinstance(value, dict): + self.assertTrue(forbidden_keys.isdisjoint(value)) + for child in value.values(): + assert_safe(child) + elif isinstance(value, list): + for child in value: + assert_safe(child) + + assert_safe(report) + self.assertNotIn("192.0.2.10", serialized) + self.assertNotIn("198.51.100.20", serialized) + self.assertNotIn("1893456000", serialized) + + def test_diagnostics_are_debug_sidecar_not_public_export_data(self): + session = diagnostic_fixture_session() + export = build_export_json([], []) + self.assertNotIn("diagnostics", export) + self.assertNotIn("capture_diagnostics", export) + + with TemporaryDirectory() as temp_dir: + diagnostics_path = new_diagnostics_path(temp_dir) + write_capture_diagnostics(diagnostics_path, session.diagnostic_report()) + written = json.loads(diagnostics_path.read_text(encoding="utf-8")) + + self.assertEqual(written, session.diagnostic_report()) + self.assertRegex(diagnostics_path.name, r"^Capture_\d{8}_\d{6}\.diagnostics\.json$") + self.assertNotIn("user", diagnostics_path.name.casefold()) From 9eae9b4a150729aadadea94a1d720c7957815f00 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Thu, 16 Jul 2026 00:50:14 +0100 Subject: [PATCH 6/8] Added Mystery Box Support Added the event specific Mystery Box support to the exporter. Closes #7 --- README.md | 6 +- docs/export-format.md | 25 ++- docs/limitations.md | 3 +- docs/mapping-updates.md | 7 + docs/packet-format.md | 14 +- mappings/items.json | 17 +- mappings/mystery_box.json | 34 +++ pyproject.toml | 2 +- src/nte_history_exporter/__init__.py | 2 +- .../adapters/mitmproxy_flows.py | 19 +- src/nte_history_exporter/cli.py | 8 +- src/nte_history_exporter/console.py | 1 + src/nte_history_exporter/constants.py | 19 +- .../decoder/mystery_box.py | 208 ++++++++++++++++++ src/nte_history_exporter/export/csv_export.py | 1 + .../export/json_export.py | 16 ++ .../live_capture/runner.py | 3 +- .../live_capture/session.py | 50 ++++- src/nte_history_exporter/mapping_update.py | 47 +++- src/nte_history_exporter/pool_mappings.py | 1 + .../DT_GashaponLotteryGlobal.json | 1 + .../DataTable/Vehicle/DT_VehicleItemData.json | 1 + tests/test_mapping_update.py | 45 ++++ tests/test_mystery_box_decoding.py | 116 ++++++++++ 24 files changed, 627 insertions(+), 19 deletions(-) create mode 100644 mappings/mystery_box.json create mode 100644 src/nte_history_exporter/decoder/mystery_box.py create mode 100644 tests/fixtures/nte_assets/DataTable/GashaponLottery/DT_GashaponLotteryGlobal.json create mode 100644 tests/fixtures/nte_assets/DataTable/Vehicle/DT_VehicleItemData.json create mode 100644 tests/test_mystery_box_decoding.py diff --git a/README.md b/README.md index a30c397..7f553b6 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,11 @@ Prototype CLI exporter for **Neverness to Everness** pull history — decodes yo | Monopoly | Standard Board | `Lottery_Permanent` | Per-banner | | Monopoly | Limited Character Board | `Lottery_LimitedCharacter` | Shared | | Gashapon | Arc Miracle Box | `Arc_MiracleBox` | Shared | +| Gashapon | Mystery Box | `Gashapon_MysteryBox` | Per-rotation | ## What It Does -The exporter decodes Permanent Board, Limited Character Board, and Arc Miracle Box history pages from captured UDP data, applies conservative timestamp-boundary handling, and writes sanitized JSON suitable for tracker import. +The exporter decodes Permanent Board, Limited Character Board, Arc Miracle Box, and Mystery Box history pages from captured UDP data, applies conservative timestamp-boundary handling, and writes sanitized JSON suitable for tracker import. > [!NOTE] > The import JSON contains decoded history rows and the shareable NTE user UID when it can be detected. It does **not** export tokens, account IDs, role IDs, device IDs, server IPs, raw packets, cookies, session data, or other capture metadata. @@ -96,6 +97,7 @@ Once running, open any supported history board in game. The tool keeps listening - `_Permanent_.json` - `_Limited_.json` - `_Arc_.json` +- `_MysteryBox_.json` If the user UID is not detected automatically, the console asks for it before saving. Leaving it blank saves as `unknown__.json`, but may prevent import on some trackers. @@ -199,7 +201,7 @@ History always loads page 1 first and is scrolled downward, so the exporter anch Within a timestamp group, ordinal 0 is the newest record and unseen rows can only append after the captured ones, so **every exported UID is stable** — including a partially captured oldest 10-pull. All decoded rows are therefore exported. Re-scanning later simply adds any rows that were not yet captured, with the same UIDs for the rows already seen. -For Monopoly, Points Gift and Chase Reward rows stay in the timestamp group for UID ordinal generation, but only `result_type = dice` rows count toward pull-set sizing. Arc pulls are always 10-pulls. In both systems every captured group is exported, including the oldest one even if it is a partially captured pull set, because its captured prefix is ordinal-stable. +For Monopoly, Points Gift and Chase Reward rows stay in the timestamp group for UID ordinal generation, but only `result_type = dice` rows count toward pull-set sizing. Arc pulls are always 10-pulls. Mystery Box records are single pulls and the final page may contain fewer than five records. Every captured group is exported, including an incomplete oldest pull set, because its captured prefix is ordinal-stable. ## Adapters diff --git a/docs/export-format.md b/docs/export-format.md index 0055174..b19e11e 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -15,7 +15,7 @@ decoder-only offsets. "capture_source": "npcap", "exporter": { "name": "nte-history-exporter", - "version": "0.1.10" + "version": "0.1.11" }, "banner": { "id": "Lottery_Permanent", @@ -71,6 +71,8 @@ Current stable pool IDs: - `Lottery_LimitedCharacter`: Limited Character Board. New limited character banners should still use this ID while they share the same history/pity pool. - `Arc_MiracleBox`: Arc Miracle Box. +- `Gashapon_MysteryBox`: Mystery Box. Records are single pulls and history is + not split into event rotations by the exporter. Avoid using `banner.name`, `reward_name`, `reward_type` or `reward_rank` as primary IDs. They are useful display fields, but may change when mapping files are updated. @@ -110,6 +112,9 @@ Arc source: nte|gashapon|pool_group_id|timestamp_raw|timestamp_group_ordinal ``` +Mystery Box uses the same Gashapon UID source. Reward quantity is preserved, +but each record represents exactly one pull and has `result_type = single_pull`. + ## Example records Monopoly: @@ -146,6 +151,24 @@ Arc: } ``` +Mystery Box: + +```json +{ + "uid": "2610c6e96afd64aa4a53fb9e8cebe031", + "pool_group_id": "Gashapon_MysteryBox", + "timestamp": "2026-07-08 19:09:33", + "timestamp_group_ordinal": 0, + "result_type": "single_pull", + "reward_type": "item", + "reward_id": "vehicle039", + "reward_name": "Draco", + "reward_rank": "S", + "quantity": 1, + "source_type": "mystery_box" +} +``` + ## CSV diagnostics CSV exports are mainly for debugging the decoder. Tools should prefer JSON diff --git a/docs/limitations.md b/docs/limitations.md index beaff11..bfd5c13 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -1,10 +1,11 @@ # Limitations -This is a prototype and supports `Lottery_Permanent`, `Lottery_LimitedCharacter`, and `Arc_MiracleBox`. +This is a prototype and supports `Lottery_Permanent`, `Lottery_LimitedCharacter`, `Arc_MiracleBox`, and `Gashapon_MysteryBox`. Known limitations: - Other NTE banners are not implemented yet. +- Mystery Box history does not expose a semantic rotation ID in each decoded row. The exporter deliberately keeps one unsplit history stream and leaves rotation grouping to consumers. - The game appears not to provide a unique server-side roll ID in the decoded record body. - UIDs are generated deterministically from decoded fields and timestamp-group order. - A partially captured oldest timestamp group is still exported; its captured prefix has stable UIDs, and a later deeper scan adds the rest with the same UIDs. diff --git a/docs/mapping-updates.md b/docs/mapping-updates.md index e4212e7..aa67d25 100644 --- a/docs/mapping-updates.md +++ b/docs/mapping-updates.md @@ -76,6 +76,13 @@ from the committed reward mappings. It still writes the staged artifacts. - Other pull rewards are selected by `GachaIllustrate` and resolved to the inventory tables for quality and localization keys. Appearance-table IDs provide canonical casing for glider rewards. +- Mystery Box rewards are selected from every row in + `DT_GashaponLotteryGlobal`. No event IDs are hard-coded, so future rotations + are included automatically. Rewards resolve through the inventory, capital, + appearance, and vehicle-item tables using case-insensitive IDs. +- Fashion, avatar-frame, and business-card Mystery Box rewards are categorized + as cosmetics; other Mystery Box rewards, including vehicle keys and + currencies, are categorized as items. - Names are resolved strictly through `Localization/en/game.json`. A missing or ambiguous key fails the update instead of falling back to a DT value. - Character-awakening illustration entries are not independent pull rewards diff --git a/docs/packet-format.md b/docs/packet-format.md index ff5df85..de1c643 100644 --- a/docs/packet-format.md +++ b/docs/packet-format.md @@ -1,6 +1,6 @@ # Packet Format Notes -This prototype supports separate Monopoly and Arc/Gashapon history decoders. +This prototype supports separate Monopoly, Arc/Gashapon, and Mystery Box history decoders. ## Decoder strategy @@ -105,3 +105,15 @@ Pages are anchored to the continuous run starting at page 1 (history always load - Arc timestamps use `unix_seconds = little_endian_u64(timestamp_raw) / 20000000 - 62135596800`. - Arc pulls are treated as 10-pull timestamp groups. Like Monopoly, every captured group is exported, including the oldest one even if the scan stopped mid-10-pull (its captured prefix is ordinal-stable). - Arc rows use the same `reward_type`, `reward_id`, `reward_name`, `reward_rank`, and `reward_key_hex` fields as Monopoly rows. + +## Mystery Box + +- Mystery Box requests use a 54-byte prefix. +- Request constant: `2060` / `0x080c` at offset 26. +- Request kind: `2110` / `0x083e` at offset 35. +- Request cursor: offset 31, with a step of `2`; page is `cursor / 2`. +- Responses contain an `FGashaponLotteryRecordData` structured block. +- Each row contains the reward ID, exact quantity, a record flag, and a standard .NET timestamp. +- Every row is one single pull regardless of its reward quantity. +- A full response page contains 5 rows; the final page may contain 1–4 rows. +- History is exported under `Gashapon_MysteryBox` without attempting to infer or split rotations. diff --git a/mappings/items.json b/mappings/items.json index f8daf7e..263f9ef 100644 --- a/mappings/items.json +++ b/mappings/items.json @@ -21,5 +21,20 @@ "Fashion_vehicle_1071_V010": {"type": "cosmetic", "name": "Hound Blitz - Livery", "rank": "S"}, "Fashion_vehicle_1075_V006": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"}, "Fashion_vehicle_1076_V024": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"}, - "Fashion_vehicleSkin_1052_V016": {"type": "cosmetic", "name": "Golden Breeze - Livery", "rank": "S"} + "Fashion_vehicleSkin_1052_V016": {"type": "cosmetic", "name": "Golden Breeze - Livery", "rank": "S"}, + "vehicle039": {"type": "item", "name": "Draco", "rank": "S"}, + "Fashion_1051_rpg_level3_Dye1": {"type": "cosmetic", "name": "Glory Days - Braveheart", "rank": "S"}, + "Fashion_1051_rpg_level3_Dye2": {"type": "cosmetic", "name": "Glory Days - Starfall", "rank": "S"}, + "Fashion_1051_rpg_level3_Base": {"type": "cosmetic", "name": "Glory Days", "rank": "S"}, + "SpecialGift_ticket": {"type": "item", "name": "Floe Cinema Ticket", "rank": "A"}, + "Fons": {"type": "item", "name": "Fons", "rank": "A"}, + "CharacterUpMaterial_lv3": {"type": "item", "name": "Elite Hunter Guide", "rank": "A"}, + "Gold": {"type": "item", "name": "Beetle Coin", "rank": "B"}, + "CityAbility_UpMaterial": {"type": "item", "name": "Dreamless Seed", "rank": "B"}, + "SpecialGift_letter": {"type": "item", "name": "A Handwritten Letter", "rank": "S"}, + "Frame_MangHe_shinku": {"type": "cosmetic", "name": "Ember Trail", "rank": "S"}, + "Fashion_character_1051_07": {"type": "cosmetic", "name": "Perfect Partners", "rank": "S"}, + "vehicle026": {"type": "item", "name": "Porsche 918 Spyder", "rank": "S"}, + "Frame_MangHe_bsj": {"type": "cosmetic", "name": "Heartbeat Moment", "rank": "S"}, + "BussinessCard_MangHe_bsj": {"type": "cosmetic", "name": "Trackside Shadow", "rank": "S"} } diff --git a/mappings/mystery_box.json b/mappings/mystery_box.json new file mode 100644 index 0000000..3f134dc --- /dev/null +++ b/mappings/mystery_box.json @@ -0,0 +1,34 @@ +{ + "pool_key": "mystery_box", + "game": "Neverness to Everness", + "system": { + "id": "gashapon", + "name": "Gashapon" + }, + "banner": { + "id": "Gashapon_MysteryBox", + "name": "Mystery Box", + "shared_pity": false + }, + "request": { + "family": "mystery_box", + "length": 54, + "constant": 2060, + "constant_offset": 26, + "request_kind": 2110, + "request_kind_offset": 35, + "cursor_offset": 31, + "cursor_step": 2, + "page_formula": "cursor / 2" + }, + "response": { + "records_per_full_page": 5, + "marker": "FGashaponLotteryRecordData", + "pulls_per_record": 1, + "final_page_may_be_partial": true + }, + "notes": [ + "Mystery Box is a finite 15-draw prize pool whose rewards are removed as they are drawn.", + "History is exported as one pull per record and is not split into rotations by the exporter." + ] +} diff --git a/pyproject.toml b/pyproject.toml index 4e8358b..edfd7cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nte-history-exporter" -version = "0.1.10" +version = "0.1.11" description = "Cross-platform Neverness to Everness pull-history exporter." requires-python = ">=3.10" dependencies = [] diff --git a/src/nte_history_exporter/__init__.py b/src/nte_history_exporter/__init__.py index 569b121..0c5c300 100644 --- a/src/nte_history_exporter/__init__.py +++ b/src/nte_history_exporter/__init__.py @@ -1 +1 @@ -__version__ = "0.1.10" +__version__ = "0.1.11" diff --git a/src/nte_history_exporter/adapters/mitmproxy_flows.py b/src/nte_history_exporter/adapters/mitmproxy_flows.py index fc2b930..3cb8d48 100644 --- a/src/nte_history_exporter/adapters/mitmproxy_flows.py +++ b/src/nte_history_exporter/adapters/mitmproxy_flows.py @@ -7,6 +7,10 @@ from typing import Any from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 from nte_history_exporter.decoder.arc import build_arc_rows_from_pairs, select_continuous_arc_run from nte_history_exporter.decoder.run import build_rows_from_pairs +from nte_history_exporter.decoder.mystery_box import ( + build_mystery_box_rows_from_pairs, + select_continuous_mystery_box_run, +) from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket @@ -101,12 +105,21 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d packet = UdpPacket(ts, remote_ip, local_ip, remote_port, local_port, content) session.process_packet(packet) - pairs = [pair for pair in session.pairs if pair[7] != "arc_miracle_box"] + pairs = [ + pair + for pair in session.pairs + if pair[7] not in {"arc_miracle_box", "mystery_box"} + ] arc_pairs = [pair for pair in session.pairs if pair[7] == "arc_miracle_box"] + mystery_box_pairs = [pair for pair in session.pairs if pair[7] == "mystery_box"] best_run, run_warnings = select_continuous_run_from_page_1(pairs) rows_out = build_rows_from_pairs(best_run) best_arc_run, arc_warnings = select_continuous_arc_run(arc_pairs) arc_rows = build_arc_rows_from_pairs(best_arc_run) + best_mystery_box_run, mystery_box_warnings = select_continuous_mystery_box_run( + mystery_box_pairs + ) + mystery_box_rows = build_mystery_box_rows_from_pairs(best_mystery_box_run) return { "flow_index": resolved_flow_index, @@ -118,6 +131,10 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d "best_arc_run": best_arc_run, "arc_rows": arc_rows, "arc_warnings": arc_warnings, + "mystery_box_pairs": mystery_box_pairs, + "best_mystery_box_run": best_mystery_box_run, + "mystery_box_rows": mystery_box_rows, + "mystery_box_warnings": mystery_box_warnings, "user_uid": session.user_uid or user_uid, "capture_diagnostics": session.diagnostic_report(), } diff --git a/src/nte_history_exporter/cli.py b/src/nte_history_exporter/cli.py index 5b0090e..d5d4bf0 100644 --- a/src/nte_history_exporter/cli.py +++ b/src/nte_history_exporter/cli.py @@ -66,7 +66,13 @@ def main(argv: list[str] | None = None) -> int: return 1 decoded = decode_mitmproxy_flows(args.capture_source, args.flow_index) - if decoded["arc_rows"] and not decoded["rows"]: + if decoded["mystery_box_rows"] and not decoded["rows"] and not decoded["arc_rows"]: + rows = decoded["mystery_box_rows"] + warnings = decoded["mystery_box_warnings"] + kind = "mystery_box" + best_run = decoded["best_mystery_box_run"] + pair_count = len(decoded["mystery_box_pairs"]) + elif decoded["arc_rows"] and not decoded["rows"]: rows = decoded["arc_rows"] warnings = decoded["arc_warnings"] kind = "arc_miracle_box" diff --git a/src/nte_history_exporter/console.py b/src/nte_history_exporter/console.py index c24a053..ecc65ba 100644 --- a/src/nte_history_exporter/console.py +++ b/src/nte_history_exporter/console.py @@ -82,6 +82,7 @@ def print_live_instructions(local_ip: str, backend: str = "windows_raw", detail: print(style(" Monopoly > Standard Board history", CYAN)) print(style(" Monopoly > Limited Character Board history", CYAN)) print(style(" Gashapon > Arc Miracle Box history", CYAN)) + print(style(" Gashapon > Mystery Box history", CYAN)) print(" 3. Start at page 1 and scroll down through every page") print(" you want exported.") print(" 4. Scroll one page past where you plan to stop so the") diff --git a/src/nte_history_exporter/constants.py b/src/nte_history_exporter/constants.py index 6e01073..300096e 100644 --- a/src/nte_history_exporter/constants.py +++ b/src/nte_history_exporter/constants.py @@ -10,8 +10,10 @@ LIMITED_CHARACTER_BANNER_ID = "Lottery_LimitedCharacter" LIMITED_CHARACTER_BANNER_NAME = "Limited Character Board" ARC_BANNER_ID = "Arc_MiracleBox" ARC_BANNER_NAME = "Arc Miracle Box" +MYSTERY_BOX_BANNER_ID = "Gashapon_MysteryBox" +MYSTERY_BOX_BANNER_NAME = "Mystery Box" EXPORTER_NAME = "nte-history-exporter" -EXPORTER_VERSION = "0.1.10" +EXPORTER_VERSION = "0.1.11" HISTORY_REQUEST_BANNER = 4220 HISTORY_REQUEST_LENGTH = 45 @@ -38,6 +40,12 @@ POOL_META = { "system": ARC_SYSTEM_NAME, "shared_pity": True, }, + "mystery_box": { + "id": MYSTERY_BOX_BANNER_ID, + "name": MYSTERY_BOX_BANNER_NAME, + "system": ARC_SYSTEM_NAME, + "shared_pity": False, + }, } ARC_HISTORY_REQUEST_BANNER = 2060 @@ -47,6 +55,15 @@ ARC_HISTORY_PAGE_CURSOR_MULTIPLIER = 2 ARC_RESPONSE_FIRST_RECORD_OFFSET = 0x4C ARC_TIMESTAMP_TICKS_PER_SECOND = 20_000_000 +MYSTERY_BOX_HISTORY_REQUEST_LENGTH = 54 +MYSTERY_BOX_HISTORY_REQUEST_CONSTANT_OFFSET = 26 +MYSTERY_BOX_HISTORY_REQUEST_CONSTANT = 2060 +MYSTERY_BOX_HISTORY_CURSOR_OFFSET = 31 +MYSTERY_BOX_HISTORY_PAGE_CURSOR_MULTIPLIER = 2 +MYSTERY_BOX_HISTORY_REQUEST_KIND_OFFSET = 35 +MYSTERY_BOX_HISTORY_REQUEST_KIND = 2110 +MYSTERY_BOX_MARKER = b"FGashaponLotteryRecordData" + TIMESTAMP_TICKS_PER_SECOND = 40_000_000 DOTNET_UNIX_EPOCH_SECONDS = 62_135_596_800 diff --git a/src/nte_history_exporter/decoder/mystery_box.py b/src/nte_history_exporter/decoder/mystery_box.py new file mode 100644 index 0000000..2c9eaf2 --- /dev/null +++ b/src/nte_history_exporter/decoder/mystery_box.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import hashlib +import struct +from collections import defaultdict +from datetime import datetime, timezone +from typing import Any, Iterator + +from nte_history_exporter.constants import ( + ARC_SYSTEM, + DOTNET_UNIX_EPOCH_SECONDS, + GAME_UID_PART, + MYSTERY_BOX_BANNER_ID, + MYSTERY_BOX_HISTORY_CURSOR_OFFSET, + MYSTERY_BOX_HISTORY_PAGE_CURSOR_MULTIPLIER, + MYSTERY_BOX_HISTORY_REQUEST_CONSTANT, + MYSTERY_BOX_HISTORY_REQUEST_CONSTANT_OFFSET, + MYSTERY_BOX_HISTORY_REQUEST_KIND, + MYSTERY_BOX_HISTORY_REQUEST_KIND_OFFSET, + MYSTERY_BOX_HISTORY_REQUEST_LENGTH, + MYSTERY_BOX_MARKER, + POOL_META, +) +from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 +from nte_history_exporter.decoder.protocol import infer_reward_type +from nte_history_exporter.decoder.run import fmt_packet_time +from nte_history_exporter.mappings import REWARDS_BY_ID + +DOTNET_TICKS_PER_SECOND = 10_000_000 +MAX_RECORDS_PER_BLOCK = 100 +MAX_REWARD_ID_LENGTH = 256 +REWARDS_BY_CASEFOLDED_ID = { + reward_id.casefold(): reward for reward_id, reward in REWARDS_BY_ID.items() +} + + +def is_mystery_box_history_request(content: bytes) -> bool: + if len(content) < MYSTERY_BOX_HISTORY_REQUEST_LENGTH: + return False + return ( + struct.unpack_from(" int: + cursor = struct.unpack_from(" Iterator[bytes]: + yield payload + # History payloads can begin at a non-byte-aligned transport position. + for bit_shift in range(1, 8): + shifted = bytearray() + for index in range(max(0, len(payload) - 8)): + bit_pos = (8 + index) * 8 + bit_shift + byte_pos, shift = divmod(bit_pos, 8) + if byte_pos >= len(payload): + break + value = payload[byte_pos] >> shift + if shift and byte_pos + 1 < len(payload): + value |= payload[byte_pos + 1] << (8 - shift) + shifted.append(value & 0xFF) + yield bytes(shifted) + + +def _decode_timestamp(raw: bytes) -> tuple[int, float, str]: + ticks = struct.unpack(" list[dict[str, Any]]: + marker_pos = data.find(MYSTERY_BOX_MARKER) + if marker_pos < 0: + return [] + pos = marker_pos + len(MYSTERY_BOX_MARKER) + if pos < len(data) and data[pos] == 0: + pos += 1 + if pos + 12 > len(data): + return [] + _reserved, _declared_size, row_count = struct.unpack_from(" MAX_RECORDS_PER_BLOCK: + return [] + + rows: list[dict[str, Any]] = [] + try: + for _ in range(row_count): + record_start = pos + reward_length = struct.unpack_from(" len(data): + return [] + reward_raw = data[pos : pos + reward_length] + pos += reward_length + if not reward_raw.endswith(b"\0"): + return [] + reward_id = reward_raw[:-1].decode("utf-8") + quantity = struct.unpack_from(" list[dict[str, Any]]: + for view in _protocol_views(payload): + rows = _parse_view(view) + if rows: + return rows + return [] + + +def make_mystery_box_uid(timestamp_raw: str, ordinal: int) -> str: + source = "|".join( + [GAME_UID_PART, ARC_SYSTEM, MYSTERY_BOX_BANNER_ID, timestamp_raw, str(ordinal)] + ) + return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32] + + +def annotate_mystery_box_rows(rows: list[dict[str, Any]]) -> None: + groups: dict[str, list[int]] = defaultdict(list) + for index, row in enumerate(rows): + groups[row["timestamp_raw_hex"]].append(index) + for group_index, (timestamp_raw, indexes) in enumerate(groups.items()): + for ordinal, index in enumerate(indexes): + row = rows[index] + row["timestamp_group_index"] = group_index + row["timestamp_group_ordinal"] = ordinal + row["timestamp_group_size_seen"] = len(indexes) + row["timestamp_group_record_size_seen"] = len(indexes) + row["timestamp_group_boundary"] = ( + "oldest" if group_index == len(groups) - 1 else ("newest" if group_index == 0 else "") + ) + row["uid"] = make_mystery_box_uid(timestamp_raw, ordinal) + row["uid_status"] = "stable" + row["export_record"] = True + row["skip_reason"] = "" + + +def build_mystery_box_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: + pool = POOL_META["mystery_box"] + rows: list[dict[str, Any]] = [] + for pair in pairs: + page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7] + records = parse_mystery_box_response(response) + if len(pair) > 9: + slice_start, slice_count = pair[8:10] + records = records[slice_start : slice_start + slice_count] + for row_index, record in enumerate(records, start=1): + rows.append( + { + **record, + "page": page, + "offset": offset, + "row": row_index, + "pool_group_id": pool["id"], + "pool_group_name": pool["name"], + "request_msg": req_i, + "request_time_utc": fmt_packet_time(req_ts), + "response_msg": resp_i, + "response_time_utc": fmt_packet_time(resp_ts), + "response_len": len(response), + "record_count": len(records), + } + ) + annotate_mystery_box_rows(rows) + return rows + + +def select_continuous_mystery_box_run( + pairs: list[tuple], +) -> tuple[list[tuple], list[dict[str, Any]]]: + return select_continuous_run_from_page_1(pairs) diff --git a/src/nte_history_exporter/export/csv_export.py b/src/nte_history_exporter/export/csv_export.py index 59e79aa..7e1757b 100644 --- a/src/nte_history_exporter/export/csv_export.py +++ b/src/nte_history_exporter/export/csv_export.py @@ -58,6 +58,7 @@ FIELDNAMES = [ "record_len", "dice_offset_in_record", "record_hex", + "record_flag", ] diff --git a/src/nte_history_exporter/export/json_export.py b/src/nte_history_exporter/export/json_export.py index 2735136..dd42d1f 100644 --- a/src/nte_history_exporter/export/json_export.py +++ b/src/nte_history_exporter/export/json_export.py @@ -9,6 +9,7 @@ from nte_history_exporter.constants import ( EXPORTER_NAME, GAME_NAME, POOL_META, + MYSTERY_BOX_BANNER_ID, ) @@ -83,6 +84,21 @@ def _record_for_export(row: dict[str, Any]) -> dict[str, Any]: "source_type": row.get("source_type"), } + if row.get("pool_group_id") == MYSTERY_BOX_BANNER_ID: + return { + "uid": row.get("uid"), + "pool_group_id": row.get("pool_group_id"), + "timestamp": row.get("timestamp_decoded"), + "timestamp_group_ordinal": row.get("timestamp_group_ordinal"), + "result_type": "single_pull", + "reward_type": row.get("reward_type"), + "reward_id": row.get("reward_id"), + "reward_name": row.get("reward_name"), + "reward_rank": row.get("reward_rank"), + "quantity": row.get("quantity"), + "source_type": row.get("source_type"), + } + return { "uid": row.get("uid"), "pool_group_id": row.get("pool_group_id", BANNER_ID), diff --git a/src/nte_history_exporter/live_capture/runner.py b/src/nte_history_exporter/live_capture/runner.py index 6c16d79..4c38bc9 100644 --- a/src/nte_history_exporter/live_capture/runner.py +++ b/src/nte_history_exporter/live_capture/runner.py @@ -23,6 +23,7 @@ EXPORT_PREFIXES = { "permanent": "Permanent", "limited_character": "Limited", "arc_miracle_box": "Arc", + "mystery_box": "MysteryBox", } CAPTURE_SOURCE_LABELS = { @@ -148,7 +149,7 @@ def run_live_capture( pairs = session.pairs_for_kind(kind) best_run, run_warnings = select_continuous_run_from_page_1(pairs) rows = session.build_rows(kind) - if kind != "arc_miracle_box": + if kind not in {"arc_miracle_box", "mystery_box"}: rows = annotate_groups(rows) warnings = run_warnings pages_seen = [p[0] for p in best_run] diff --git a/src/nte_history_exporter/live_capture/session.py b/src/nte_history_exporter/live_capture/session.py index 171e782..5acaae3 100644 --- a/src/nte_history_exporter/live_capture/session.py +++ b/src/nte_history_exporter/live_capture/session.py @@ -20,7 +20,15 @@ from nte_history_exporter.decoder.protocol import ( response_contains_history_marker, ) from nte_history_exporter.decoder.run import build_rows_from_pairs +from nte_history_exporter.decoder.mystery_box import ( + build_mystery_box_rows_from_pairs, + is_mystery_box_history_request, + mystery_box_request_page, + parse_mystery_box_response, + select_continuous_mystery_box_run, +) from nte_history_exporter.decoder.structured_protocol import FORK_MARKER, MONOPOLY_MARKER +from nte_history_exporter.constants import MYSTERY_BOX_MARKER from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates from nte_history_exporter.live_capture.diagnostics import CaptureDiagnostics @@ -165,6 +173,30 @@ class LiveHistorySession: self.last_page_seen = req.page return False + if packet.src_ip == self.local_ip and is_mystery_box_history_request(packet.payload): + page = mystery_box_request_page(packet.payload) + req = PendingRequest( + page=page, + offset=page * 2, + kind="mystery_box", + request_msg=self.packet_count, + request_time=packet.timestamp, + src_ip=packet.src_ip, + dst_ip=packet.dst_ip, + src_port=packet.src_port, + dst_port=packet.dst_port, + ) + self._queue_request(req) + self.diagnostics.counters["history_requests_recognized"] += 1 + self.diagnostics.add_event( + "HISTORY_REQUEST_RECOGNIZED", + self.packet_count, + kind=req.kind, + page=req.page, + ) + self.last_page_seen = req.page + return False + if packet.dst_ip != self.local_ip: return False @@ -191,12 +223,24 @@ class LiveHistorySession: monopoly_records = decode_response_records(packet.payload) arc_records = parse_arc_response(packet.payload) if not monopoly_records else [] + mystery_box_records = ( + parse_mystery_box_response(packet.payload) + if not monopoly_records and not arc_records + else [] + ) if monopoly_records: - candidates = [req for req in connection_candidates if req.kind != "arc_miracle_box"] + candidates = [ + req + for req in connection_candidates + if req.kind not in {"arc_miracle_box", "mystery_box"} + ] records = monopoly_records elif arc_records: candidates = [req for req in connection_candidates if req.kind == "arc_miracle_box"] records = arc_records + elif mystery_box_records: + candidates = [req for req in connection_candidates if req.kind == "mystery_box"] + records = mystery_box_records else: candidates = connection_candidates records = [] @@ -215,6 +259,7 @@ class LiveHistorySession: if response_contains_history_marker(packet.payload) or MONOPOLY_MARKER in packet.payload or FORK_MARKER in packet.payload + or MYSTERY_BOX_MARKER in packet.payload else "NO_HISTORY_MARKER" ) self._record_rejected_candidate(candidates, packet.payload, reason) @@ -340,6 +385,9 @@ class LiveHistorySession: if kind == "arc_miracle_box": best_run, _warnings = select_continuous_arc_run(self.pairs_for_kind(kind)) return build_arc_rows_from_pairs(best_run) + if kind == "mystery_box": + best_run, _warnings = select_continuous_mystery_box_run(self.pairs_for_kind(kind)) + return build_mystery_box_rows_from_pairs(best_run) return build_rows_from_pairs(self.best_run(kind)) def best_run(self, kind: str | None = None) -> list[tuple]: diff --git a/src/nte_history_exporter/mapping_update.py b/src/nte_history_exporter/mapping_update.py index e231c34..8ff2986 100644 --- a/src/nte_history_exporter/mapping_update.py +++ b/src/nte_history_exporter/mapping_update.py @@ -19,6 +19,8 @@ ASSET_TABLES = { "capital_inventory": "DataTable/Inventory/DT_CapitalItemConfig.json", "appearances": "DataTable/Character/Appearance/DT_AppearanceData.json", "illustrations": "DataTable/Gacha/GachaIllustrate.json", + "mystery_box_pools": "DataTable/GashaponLottery/DT_GashaponLotteryGlobal.json", + "vehicle_inventory": "DataTable/Vehicle/DT_VehicleItemData.json", "localization": "Localization/en/game.json", } REWARD_MAPPING_FILES = ("arcs.json", "characters.json", "items.json") @@ -282,27 +284,60 @@ def _build_item_mapping( inventory = _casefold_index(tables["inventory"], "inventory") capital = _casefold_index(tables["capital_inventory"], "capital inventory") appearances = _casefold_index(tables["appearances"], "appearances") + vehicle_inventory = _casefold_index(tables["vehicle_inventory"], "vehicle inventory") result: dict[str, dict[str, Any]] = {} - for illustrated_id in tables["illustrations"]: - folded = illustrated_id.casefold() - if illustrated_id.isdigit() or folded.startswith("fork_") or folded.startswith("characterawaken_"): + candidate_ids = _reward_candidate_ids(tables) + for candidate_id in candidate_ids: + folded = candidate_id.casefold() + if candidate_id.isdigit() or folded.startswith("fork_") or folded.startswith("characterawaken_"): continue - sources = (inventory, capital, appearances) + sources = (inventory, capital, appearances, vehicle_inventory) match = next((source.get(folded) for source in sources if folded in source), None) if match is None: - raise MappingUpdateError(f"illustrated reward {illustrated_id} is missing from item tables") + raise MappingUpdateError(f"reward {candidate_id} is missing from item tables") canonical_id, row = match if folded.startswith("fashion_glide_") and folded in appearances: canonical_id = appearances[folded][0] meta = _normalise_row(canonical_id, row, translations) result[canonical_id] = { - "type": "cosmetic" if canonical_id.casefold().startswith("fashion_") else "item", + "type": _item_mapping_type(canonical_id), "name": meta["name"], "rank": meta["rank"], } return dict(sorted(result.items(), key=lambda pair: (pair[0].casefold(), pair[0]))) +def _reward_candidate_ids(tables: dict[str, dict[str, Any]]) -> list[str]: + """Return rewards used by ordinary Gacha and every Mystery Box rotation.""" + candidates: dict[str, str] = { + item_id.casefold(): item_id for item_id in tables["illustrations"] + } + for pool_id, pool in tables["mystery_box_pools"].items(): + if not isinstance(pool, dict): + raise MappingUpdateError(f"Mystery Box pool {pool_id} must be an object") + gifts = pool.get("GiftList") + if not isinstance(gifts, list): + raise MappingUpdateError(f"Mystery Box pool {pool_id} has no GiftList") + for index, gift in enumerate(gifts): + if not isinstance(gift, dict): + raise MappingUpdateError( + f"Mystery Box pool {pool_id} gift {index} must be an object" + ) + item_id = gift.get("ItemID") + if not isinstance(item_id, str) or not item_id: + raise MappingUpdateError( + f"Mystery Box pool {pool_id} gift {index} has no ItemID" + ) + candidates.setdefault(item_id.casefold(), item_id) + return sorted(candidates.values(), key=lambda value: (value.casefold(), value)) + + +def _item_mapping_type(item_id: str) -> str: + folded = item_id.casefold() + cosmetic_prefixes = ("fashion_", "frame_", "bussinesscard_") + return "cosmetic" if folded.startswith(cosmetic_prefixes) else "item" + + def _casefold_index(rows: dict[str, Any], label: str) -> dict[str, tuple[str, Any]]: result: dict[str, tuple[str, Any]] = {} for item_id, row in rows.items(): diff --git a/src/nte_history_exporter/pool_mappings.py b/src/nte_history_exporter/pool_mappings.py index acda87e..0c47e76 100644 --- a/src/nte_history_exporter/pool_mappings.py +++ b/src/nte_history_exporter/pool_mappings.py @@ -8,6 +8,7 @@ POOL_MAPPING_FILES = { "permanent": "permanent_board.json", "limited_character": "limited_character_board.json", "arc_miracle_box": "arc_miracle_box.json", + "mystery_box": "mystery_box.json", } diff --git a/tests/fixtures/nte_assets/DataTable/GashaponLottery/DT_GashaponLotteryGlobal.json b/tests/fixtures/nte_assets/DataTable/GashaponLottery/DT_GashaponLotteryGlobal.json new file mode 100644 index 0000000..edf0e46 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/GashaponLottery/DT_GashaponLotteryGlobal.json @@ -0,0 +1 @@ +[{"Rows":{}}] diff --git a/tests/fixtures/nte_assets/DataTable/Vehicle/DT_VehicleItemData.json b/tests/fixtures/nte_assets/DataTable/Vehicle/DT_VehicleItemData.json new file mode 100644 index 0000000..edf0e46 --- /dev/null +++ b/tests/fixtures/nte_assets/DataTable/Vehicle/DT_VehicleItemData.json @@ -0,0 +1 @@ +[{"Rows":{}}] diff --git a/tests/test_mapping_update.py b/tests/test_mapping_update.py index a715f16..71c59cb 100644 --- a/tests/test_mapping_update.py +++ b/tests/test_mapping_update.py @@ -72,6 +72,51 @@ class MappingUpdateTests(unittest.TestCase): self.assertNotIn("UnusedItem", result.mappings["items.json"]) self.assertFalse(any(key.startswith("Characterawaken_") for key in result.mappings["items.json"])) + def test_future_mystery_box_pool_is_discovered_without_hard_coded_event_id(self): + assets = load_assets(assets_root=SAMPLE_ASSETS) + tables = deepcopy(assets.tables) + tables["mystery_box_pools"]["MangHe_wowzers"] = { + "GiftList": [ + {"ItemID": "WowTicket"}, + {"ItemID": "Frame_Wowzers"}, + {"ItemID": "VehicleWow"}, + ] + } + tables["inventory"]["WowTicket"] = { + "ItemName": {"TableId": "/Game/Text/ST_Item.ST_Item", "Key": "wow_ticket"}, + "ItemQuality": "EItemQuality::ITEM_QUALITY_PURPLE", + } + tables["inventory"]["Frame_Wowzers"] = { + "ItemName": {"TableId": "/Game/Text/ST_Item.ST_Item", "Key": "wow_frame"}, + "ItemQuality": "EItemQuality::ITEM_QUALITY_BLUE", + } + tables["vehicle_inventory"]["vehiclewow"] = { + "ItemName": { + "TableId": "/Game/Text/ST_VehicleData.ST_VehicleData", + "Key": "vehicle_wow", + }, + "ItemQuality": "EItemQuality::ITEM_QUALITY_ORANGE", + } + tables["localization"]["ST_Item"].update( + {"wow_ticket": "Wowzers Ticket", "wow_frame": "Wowzers Frame"} + ) + tables["localization"]["ST_VehicleData"] = {"vehicle_wow": "Wowmobile"} + + result = build_mapping_update(sample_current(), replace(assets, tables=tables)) + + self.assertEqual( + result.mappings["items.json"]["WowTicket"], + {"type": "item", "name": "Wowzers Ticket", "rank": "A"}, + ) + self.assertEqual( + result.mappings["items.json"]["Frame_Wowzers"], + {"type": "cosmetic", "name": "Wowzers Frame", "rank": "B"}, + ) + self.assertEqual( + result.mappings["items.json"]["vehiclewow"], + {"type": "item", "name": "Wowmobile", "rank": "S"}, + ) + def test_embedded_localized_strings_are_ignored(self): _current, result = build_sample_update() diff --git a/tests/test_mystery_box_decoding.py b/tests/test_mystery_box_decoding.py new file mode 100644 index 0000000..6d0e7e1 --- /dev/null +++ b/tests/test_mystery_box_decoding.py @@ -0,0 +1,116 @@ +from tests.support import * # noqa: F401,F403 + +from datetime import datetime, timezone + +from nte_history_exporter.constants import MYSTERY_BOX_MARKER +from nte_history_exporter.decoder.mystery_box import ( + build_mystery_box_rows_from_pairs, + is_mystery_box_history_request, + mystery_box_request_page, + parse_mystery_box_response, +) + + +def mystery_box_request(page: int) -> bytes: + request = bytearray(54) + request[26:30] = (2060).to_bytes(4, "little") + request[31:35] = (page * 2).to_bytes(4, "little") + request[35:39] = (2110).to_bytes(4, "little") + request[40:44] = (2).to_bytes(4, "little") + request[44:48] = (2060).to_bytes(4, "little") + request[49:53] = (10).to_bytes(4, "little") + request[53] = 6 + return bytes(request) + + +def mystery_box_response(records: list[tuple[str, int, datetime]]) -> bytes: + body = bytearray() + for reward_id, quantity, timestamp in records: + encoded_id = reward_id.encode("utf-8") + b"\0" + ticks = int(timestamp.timestamp() * 10_000_000) + 621_355_968_000_000_000 + body += len(encoded_id).to_bytes(4, "little") + body += encoded_id + body += quantity.to_bytes(4, "little") + body += b"\x01" + body += ticks.to_bytes(8, "little") + return ( + bytes(50) + + MYSTERY_BOX_MARKER + + b"\0" + + bytes(4) + + len(body).to_bytes(4, "little") + + len(records).to_bytes(4, "little") + + body + + b"\x03" + ) + + +class MysteryBoxDecodingTests(unittest.TestCase): + def test_recognizes_request_and_page_cursor(self): + request = mystery_box_request(6) + self.assertTrue(is_mystery_box_history_request(request)) + self.assertEqual(mystery_box_request_page(request), 6) + + def test_decodes_exact_quantity_and_timestamp(self): + timestamp = datetime(2026, 7, 8, 19, 9, 33, tzinfo=timezone.utc) + rows = parse_mystery_box_response( + mystery_box_response( + [ + ("vehicle039", 1, timestamp), + ("SpecialGift_ticket", 3, timestamp), + ("gold", 100_000, timestamp), + ] + ) + ) + self.assertEqual(len(rows), 3) + self.assertEqual(rows[0]["reward_id"], "vehicle039") + self.assertEqual(rows[0]["reward_name"], "Draco") + self.assertEqual(rows[0]["timestamp_decoded"], "2026-07-08 19:09:33") + self.assertEqual(rows[1]["quantity"], 3) + self.assertEqual(rows[1]["result_type"], "single_pull") + self.assertEqual(rows[2]["reward_name"], "Beetle Coin") + self.assertEqual(rows[2]["reward_rank"], "B") + + def test_live_session_accepts_partial_final_page(self): + session = LiveHistorySession("192.168.0.10") + request = mystery_box_request(3) + timestamp = datetime(2026, 7, 8, 19, 9, 33, tzinfo=timezone.utc) + response = mystery_box_response( + [("Fons", 100_000, timestamp), ("gold", 100_000, timestamp)] + ) + self.assertFalse( + session.process_packet( + UdpPacket(1.0, "192.168.0.10", "203.0.113.5", 50000, 40000, request) + ) + ) + self.assertTrue( + session.process_packet( + UdpPacket(1.1, "203.0.113.5", "192.168.0.10", 40000, 50000, response) + ) + ) + self.assertEqual(session.kinds_seen(), ["mystery_box"]) + self.assertEqual(session.pairs[0][0], 3) + self.assertEqual(session.pairs[0][8:10], (0, 2)) + + def test_rows_and_export_use_non_shared_mystery_box_banner(self): + timestamp = datetime(2026, 7, 8, 19, 9, 33, tzinfo=timezone.utc) + response = mystery_box_response([("Fons", 100_000, timestamp)]) + rows = build_mystery_box_rows_from_pairs( + [(1, 2, 1, 1.0, 2, 1.1, response, "mystery_box", 0, 1)] + ) + export = build_export_json(rows, []) + self.assertEqual(export["banner"]["id"], "Gashapon_MysteryBox") + self.assertEqual(export["banner"]["name"], "Mystery Box") + self.assertIs(export["banner"]["shared_pity"], False) + self.assertEqual(export["records"][0]["result_type"], "single_pull") + self.assertEqual(export["records"][0]["quantity"], 100_000) + self.assertNotIn("roll_result", export["records"][0]) + self.assertEqual(rows[0]["timestamp_group_size_seen"], 1) + self.assertEqual(rows[0]["uid_status"], "stable") + + def test_export_path_uses_mystery_box_prefix(self): + _csv_path, json_path = export_paths("mystery_box", "218216016349") + self.assertRegex( + json_path.name, + r"^218216016349_MysteryBox_\d{8}_\d{6}(?:_\d+)?\.json$", + ) From cc88955baf68702e7c47ea8b27f7596d637b22a2 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Thu, 16 Jul 2026 01:34:38 +0100 Subject: [PATCH 7/8] Added server detection Implemented server detection at the request of NTEWIZ. If not detected, it will ask after collecting pull data. --- README.md | 9 ++- docs/export-format.md | 9 ++- pyproject.toml | 2 +- src/nte_history_exporter/__init__.py | 2 +- .../adapters/mitmproxy_flows.py | 7 ++ src/nte_history_exporter/cli.py | 5 ++ src/nte_history_exporter/console.py | 29 ++++++++ src/nte_history_exporter/constants.py | 2 +- .../decoder/server_region.py | 55 ++++++++++++++ .../export/json_export.py | 8 +++ .../live_capture/runner.py | 4 ++ .../live_capture/session.py | 7 ++ .../live_capture/stop_key.py | 37 +++++++++- tests/support.py | 2 + tests/test_export_contract.py | 71 +++++++++++++++++++ 15 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 src/nte_history_exporter/decoder/server_region.py diff --git a/README.md b/README.md index 7f553b6..48d9e6b 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,9 @@ From source: Or simply double-click **`run-exporter.cmd`**. > [!IMPORTANT] -> For automatic user UID detection, launch the tool **before pressing Start on the game's main menu**. If you are already in game, history capture can still work; the tool will ask for your UID before saving if it cannot detect it automatically. +> For automatic user UID and account-server detection, launch the tool **before pressing Start on the game's main menu**. If you are already in game, history capture can still work; the tool will ask for missing account details before saving. -Once running, open any supported history board in game. The tool keeps listening until you press any key. Exports are written under `exports\` as: +Once running, open any supported history board in game. The tool keeps listening until you press any key. After it prints and saves the results, press any key again to close the exporter. Exports are written under `exports\` as: - `_Permanent_.json` - `_Limited_.json` @@ -101,6 +101,11 @@ Once running, open any supported history board in game. The tool keeps listening If the user UID is not detected automatically, the console asks for it before saving. Leaving it blank saves as `unknown__.json`, but may prevent import on some trackers. +The export also includes `server_id` and `account_region` when known. If the +initial TCP server-selection response was missed, the console offers Asia, +America, Europe, and SEA as a numbered choice. The prompt can be left blank to +omit server information. + Exports are not copied to the clipboard by default. Add `--copy-clipboard` to copy a single captured banner's JSON after saving. If multiple banners are captured in the same run, clipboard copy is skipped so one banner does not overwrite another. If a page response is missed, the exporter reports the missing page number while capture is still running. Leave the exporter open, close and reopen that history board, then scroll down again. Scrolling backward within the existing view does not request the cached pages again. The replacement capture is accepted and the tool confirms when the gap has been recovered. If reopening the board still produces no page messages, return to the main menu and re-enter the game to start a fresh connection. diff --git a/docs/export-format.md b/docs/export-format.md index b19e11e..1b9ea37 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -15,7 +15,7 @@ decoder-only offsets. "capture_source": "npcap", "exporter": { "name": "nte-history-exporter", - "version": "0.1.11" + "version": "0.1.12" }, "banner": { "id": "Lottery_Permanent", @@ -32,6 +32,8 @@ decoder-only offsets. "warnings": [] }, "user_uid": "optional-user-uid", + "server_id": "23003", + "account_region": "EU", "records": [] } ``` @@ -45,6 +47,11 @@ Top-level fields: - `banner`: The history pool this file belongs to. - `scan`: Export counts and capture warnings. - `user_uid`: Optional game account UID, if known. +- `server_id`: Optional numeric account-server ID, detected from the initial TCP + connection or selected by the user. +- `account_region`: Region code mapped from a known production `server_id`. + Current values are `AS`, `NA_SA`, `EU`, and `SE`. It is omitted for an + unrecognized server ID rather than guessed. - `records`: Pull/reward records. ## Fields to identify pulls diff --git a/pyproject.toml b/pyproject.toml index edfd7cd..f87550c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nte-history-exporter" -version = "0.1.11" +version = "0.1.12" description = "Cross-platform Neverness to Everness pull-history exporter." requires-python = ">=3.10" dependencies = [] diff --git a/src/nte_history_exporter/__init__.py b/src/nte_history_exporter/__init__.py index 0c5c300..74acd0e 100644 --- a/src/nte_history_exporter/__init__.py +++ b/src/nte_history_exporter/__init__.py @@ -1 +1 @@ -__version__ = "0.1.11" +__version__ = "0.1.12" diff --git a/src/nte_history_exporter/adapters/mitmproxy_flows.py b/src/nte_history_exporter/adapters/mitmproxy_flows.py index 3cb8d48..b7500ac 100644 --- a/src/nte_history_exporter/adapters/mitmproxy_flows.py +++ b/src/nte_history_exporter/adapters/mitmproxy_flows.py @@ -12,6 +12,7 @@ from nte_history_exporter.decoder.mystery_box import ( select_continuous_mystery_box_run, ) from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates +from nte_history_exporter.decoder.server_region import extract_server_id from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket @@ -84,10 +85,15 @@ def find_udp_flow(flows: list[Any], preferred_index: int | None = None) -> tuple def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> dict[str, Any]: flows = read_flows(path) user_uid_candidates: Counter[str] = Counter() + server_id_candidates: Counter[str] = Counter() for flow in flows: for msg in flow.get(b"messages", []): user_uid_candidates.update(extract_user_uid_candidates(msg[1])) + server_id = extract_server_id(msg[1]) + if server_id: + server_id_candidates.update([server_id]) user_uid = user_uid_candidates.most_common(1)[0][0] if user_uid_candidates else None + server_id = server_id_candidates.most_common(1)[0][0] if server_id_candidates else None resolved_flow_index, flow = find_udp_flow(flows, flow_index) messages = flow[b"messages"] @@ -136,5 +142,6 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d "mystery_box_rows": mystery_box_rows, "mystery_box_warnings": mystery_box_warnings, "user_uid": session.user_uid or user_uid, + "server_id": session.server_id or server_id, "capture_diagnostics": session.diagnostic_report(), } diff --git a/src/nte_history_exporter/cli.py b/src/nte_history_exporter/cli.py index d5d4bf0..a02ed86 100644 --- a/src/nte_history_exporter/cli.py +++ b/src/nte_history_exporter/cli.py @@ -57,6 +57,7 @@ def main(argv: list[str] | None = None) -> int: write_debug_csv=args.debug, user_uid=args.user_uid, ) + console.wait_for_close() return 0 except (LibpcapUnavailable, PermissionError) as exc: console.print_problem(str(exc)) @@ -88,6 +89,9 @@ def main(argv: list[str] | None = None) -> int: resolved_user_uid = args.user_uid or decoded.get("user_uid") if not resolved_user_uid: resolved_user_uid = console.prompt_user_uid() + resolved_server_id = decoded.get("server_id") + if not resolved_server_id: + resolved_server_id = console.prompt_server_id() out_path, json_path = export_paths(kind, resolved_user_uid) diagnostics_path = None @@ -101,6 +105,7 @@ def main(argv: list[str] | None = None) -> int: source="packet_capture", capture_source="mitmproxy_flows", user_uid=resolved_user_uid, + server_id=resolved_server_id, flow_index=decoded["flow_index"], candidate_request_response_pairs=pair_count, pages_seen=[p[0] for p in best_run], diff --git a/src/nte_history_exporter/console.py b/src/nte_history_exporter/console.py index ecc65ba..10bb086 100644 --- a/src/nte_history_exporter/console.py +++ b/src/nte_history_exporter/console.py @@ -4,6 +4,8 @@ import os import sys from nte_history_exporter.constants import EXPORTER_VERSION, GAME_NAME +from nte_history_exporter.decoder.server_region import SERVER_REGIONS +from nte_history_exporter.live_capture.stop_key import wait_for_keypress WIDTH = 58 REPOSITORY_URL = "https://github.com/Golumpa/nte-exporter" @@ -179,3 +181,30 @@ def prompt_user_uid() -> str | None: print_note("Leaving this blank may prevent import on some trackers.") value = input(" User UID: ").strip() return value or None + + +def prompt_server_id() -> str | None: + print() + print_problem("Account server was not detected in this capture.") + print_note("Choose the server used by this account:") + choices = tuple( + (str(index), server_id, details["name"]) + for index, (server_id, details) in enumerate(SERVER_REGIONS.items(), start=1) + ) + for key, _server_id, name in choices: + print(f" {key}. {name}") + print_note("Leave this blank to omit server information.") + by_key = {key: server_id for key, server_id, _name in choices} + while True: + value = input(" Server [1-4]: ").strip() + if not value: + return None + if value in by_key: + return by_key[value] + print_problem("Enter 1, 2, 3, or 4; or leave it blank.") + + +def wait_for_close() -> None: + print() + print_success("Press any key to close the exporter.") + wait_for_keypress() diff --git a/src/nte_history_exporter/constants.py b/src/nte_history_exporter/constants.py index 300096e..0491f89 100644 --- a/src/nte_history_exporter/constants.py +++ b/src/nte_history_exporter/constants.py @@ -13,7 +13,7 @@ ARC_BANNER_NAME = "Arc Miracle Box" MYSTERY_BOX_BANNER_ID = "Gashapon_MysteryBox" MYSTERY_BOX_BANNER_NAME = "Mystery Box" EXPORTER_NAME = "nte-history-exporter" -EXPORTER_VERSION = "0.1.11" +EXPORTER_VERSION = "0.1.12" HISTORY_REQUEST_BANNER = 4220 HISTORY_REQUEST_LENGTH = 45 diff --git a/src/nte_history_exporter/decoder/server_region.py b/src/nte_history_exporter/decoder/server_region.py new file mode 100644 index 0000000..8106a58 --- /dev/null +++ b/src/nte_history_exporter/decoder/server_region.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import ipaddress +import struct + + +# Production entries from clientRes/OB-NTE-ServerList/serverlist_hhOS.json. +# Test and certification servers are intentionally excluded. +SERVER_REGIONS = { + "23001": {"account_region": "AS", "name": "Asia"}, + "23002": {"account_region": "NA_SA", "name": "America"}, + "23003": {"account_region": "EU", "name": "Europe"}, + "23004": {"account_region": "SE", "name": "SEA"}, +} + + +def account_region_for_server(server_id: str | None) -> str | None: + if not server_id: + return None + server = SERVER_REGIONS.get(str(server_id).strip()) + return server["account_region"] if server else None + + +def extract_server_id(payload: bytes) -> str | None: + """Extract DistrictId from the initial server-selection TCP response. + + This response is a length-prefixed FlatBuffers message. DistrictId is at + byte 96 in the current schema. Validate the frame and its following server + address before accepting the value so an arbitrary integer at that offset + is not mistaken for a server ID. + """ + if len(payload) < 168: + return None + + frame_size = struct.unpack_from(" len(payload) or frame_size < 168: + return None + if struct.unpack_from(" frame_size: + return None + try: + address = payload[136:address_end].decode("ascii") + ipaddress.ip_address(address) + except (UnicodeDecodeError, ValueError): + return None + + return str(server_id) diff --git a/src/nte_history_exporter/export/json_export.py b/src/nte_history_exporter/export/json_export.py index dd42d1f..774eef9 100644 --- a/src/nte_history_exporter/export/json_export.py +++ b/src/nte_history_exporter/export/json_export.py @@ -11,6 +11,7 @@ from nte_history_exporter.constants import ( POOL_META, MYSTERY_BOX_BANNER_ID, ) +from nte_history_exporter.decoder.server_region import account_region_for_server def build_export_json( @@ -20,6 +21,7 @@ def build_export_json( source: str = "packet_capture", capture_source: str | None = None, user_uid: str | None = None, + server_id: str | None = None, flow_index: int | None = None, candidate_request_response_pairs: int | None = None, pages_seen: list[int] | None = None, @@ -44,6 +46,7 @@ def build_export_json( scan["pages_seen"] = pages_seen normalized_user_uid = user_uid.strip() if user_uid else "" + normalized_server_id = str(server_id).strip() if server_id else "" export: dict[str, Any] = { "format": "nte-history-export", "format_version": 1, @@ -66,6 +69,11 @@ def build_export_json( ) if normalized_user_uid: export["user_uid"] = normalized_user_uid + if normalized_server_id: + export["server_id"] = normalized_server_id + account_region = account_region_for_server(normalized_server_id) + if account_region: + export["account_region"] = account_region export["records"] = [_record_for_export(r) for r in exported] return export diff --git a/src/nte_history_exporter/live_capture/runner.py b/src/nte_history_exporter/live_capture/runner.py index 4c38bc9..06e0914 100644 --- a/src/nte_history_exporter/live_capture/runner.py +++ b/src/nte_history_exporter/live_capture/runner.py @@ -144,6 +144,9 @@ def run_live_capture( resolved_user_uid = user_uid or session.user_uid if session.kinds_seen() and not resolved_user_uid: resolved_user_uid = console.prompt_user_uid() + resolved_server_id = session.server_id + if session.kinds_seen() and not resolved_server_id: + resolved_server_id = console.prompt_server_id() capture_source = CAPTURE_SOURCE_LABELS.get(capture.name, capture.name) for kind in session.kinds_seen(): pairs = session.pairs_for_kind(kind) @@ -162,6 +165,7 @@ def run_live_capture( source="live_capture", capture_source=capture_source, user_uid=resolved_user_uid, + server_id=resolved_server_id, pages_seen=pages_seen, ) payload = json.dumps(export, ensure_ascii=False, indent=2) diff --git a/src/nte_history_exporter/live_capture/session.py b/src/nte_history_exporter/live_capture/session.py index 5acaae3..0187654 100644 --- a/src/nte_history_exporter/live_capture/session.py +++ b/src/nte_history_exporter/live_capture/session.py @@ -30,6 +30,7 @@ from nte_history_exporter.decoder.mystery_box import ( from nte_history_exporter.decoder.structured_protocol import FORK_MARKER, MONOPOLY_MARKER from nte_history_exporter.constants import MYSTERY_BOX_MARKER from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates +from nte_history_exporter.decoder.server_region import extract_server_id from nte_history_exporter.live_capture.diagnostics import CaptureDiagnostics @@ -72,6 +73,7 @@ class LiveHistorySession: self.unanswered_pages: dict[str, dict[int, str]] = {} self.user_uid: str | None = None self.user_uid_candidates: Counter[str] = Counter() + self.server_id: str | None = None self.diagnostics = CaptureDiagnostics() def _mark_unanswered(self, request: PendingRequest) -> None: @@ -122,6 +124,11 @@ class LiveHistorySession: if candidates: self.user_uid_candidates.update(candidates) self.user_uid = self.user_uid_candidates.most_common(1)[0][0] + if packet.protocol == "tcp" and packet.dst_ip == self.local_ip and not self.server_id: + self.server_id = extract_server_id(packet.payload) + if self.server_id: + self.diagnostics.counters["server_ids_detected"] += 1 + self.diagnostics.add_event("SERVER_ID_DETECTED", self.packet_count) if packet.protocol != "udp": return False diff --git a/src/nte_history_exporter/live_capture/stop_key.py b/src/nte_history_exporter/live_capture/stop_key.py index c97f2cd..c6e1daa 100644 --- a/src/nte_history_exporter/live_capture/stop_key.py +++ b/src/nte_history_exporter/live_capture/stop_key.py @@ -5,6 +5,41 @@ import select import sys +def _read_windows_key() -> None: + import msvcrt + + key = msvcrt.getch() + # Function and arrow keys are emitted as a two-byte sequence. Consume the + # second byte so it cannot immediately satisfy the next key prompt. + if key in {b"\x00", b"\xe0"}: + msvcrt.getch() + + +def wait_for_keypress() -> None: + """Wait for one key on an interactive terminal, or Enter on redirected input.""" + if os.name == "nt" and sys.stdin.isatty(): + _read_windows_key() + return + + if not sys.stdin.isatty(): + try: + input() + except EOFError: + pass + return + + import termios + import tty + + fd = sys.stdin.fileno() + original_terminal = termios.tcgetattr(fd) + try: + tty.setcbreak(fd) + os.read(fd, 1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, original_terminal) + + class StopKeyMonitor: def __init__(self) -> None: self._windows = os.name == "nt" @@ -29,7 +64,7 @@ class StopKeyMonitor: if not msvcrt.kbhit(): return False - msvcrt.getch() + _read_windows_key() return True if self._fd is None: diff --git a/tests/support.py b/tests/support.py index 58a9a55..122df15 100644 --- a/tests/support.py +++ b/tests/support.py @@ -21,6 +21,7 @@ from nte_history_exporter.constants import POOL_META from nte_history_exporter.mappings import ARC_META, CHARACTERS, ITEMS, REWARDS_BY_ID from nte_history_exporter.decoder.protocol import decode_reward_key, infer_reward_type from nte_history_exporter.decoder.user_uid import extract_user_uid +from nte_history_exporter.decoder.server_region import extract_server_id from nte_history_exporter.export.csv_export import write_csv from nte_history_exporter.decoder.arc import ( arc_request_page, @@ -46,6 +47,7 @@ from nte_history_exporter.live_capture.windows_raw import parse_ipv4_packet from nte_history_exporter.live_capture.backends import open_capture_backend from nte_history_exporter.export.json_export import build_export_json from nte_history_exporter.live_capture.runner import export_paths +from nte_history_exporter import console from nte_history_exporter.pool_mappings import load_pool_mappings, pool_meta_from_mapping from nte_history_exporter.update_check import UpdateInfo, check_for_update, is_newer_version diff --git a/tests/test_export_contract.py b/tests/test_export_contract.py index 24cc919..24441ed 100644 --- a/tests/test_export_contract.py +++ b/tests/test_export_contract.py @@ -10,6 +10,8 @@ class ExportContractTests(unittest.TestCase): self.assertEqual(export["format"], "nte-history-export") self.assertIn("exporter", export) self.assertNotIn("user_uid", export) + self.assertNotIn("server_id", export) + self.assertNotIn("account_region", export) self.assertNotIn("record_hex", export["records"][0]) self.assertNotIn("request_msg", export["records"][0]) self.assertNotIn("response_msg", export["records"][0]) @@ -27,6 +29,75 @@ class ExportContractTests(unittest.TestCase): self.assertEqual(export["capture_source"], "npcap") self.assertEqual(export["user_uid"], "123456789") + def test_export_includes_server_id_and_mapped_account_region(self): + annotated = annotate_groups(fixture_session().build_rows("permanent")) + export = build_export_json(annotated, [], server_id="23003") + + self.assertEqual(export["server_id"], "23003") + self.assertEqual(export["account_region"], "EU") + + def test_export_preserves_unknown_server_without_guessing_region(self): + annotated = annotate_groups(fixture_session().build_rows("permanent")) + export = build_export_json(annotated, [], server_id="23999") + + self.assertEqual(export["server_id"], "23999") + self.assertNotIn("account_region", export) + + def test_extracts_server_id_from_valid_initial_tcp_response(self): + payload = bytearray(204) + payload[0:4] = (200).to_bytes(4, "little") + payload[4:8] = (20).to_bytes(4, "little") + payload[96:100] = (23003).to_bytes(4, "little") + address = b"198.51.100.20" + payload[132:136] = len(address).to_bytes(4, "little") + payload[136 : 136 + len(address)] = address + + self.assertEqual(extract_server_id(bytes(payload)), "23003") + + def test_rejects_server_id_at_offset_without_valid_message_structure(self): + payload = bytearray(204) + payload[96:100] = (23003).to_bytes(4, "little") + + self.assertIsNone(extract_server_id(bytes(payload))) + + def test_live_session_detects_server_only_on_inbound_tcp(self): + payload = bytearray(204) + payload[0:4] = (200).to_bytes(4, "little") + payload[4:8] = (20).to_bytes(4, "little") + payload[96:100] = (23004).to_bytes(4, "little") + address = b"198.51.100.20" + payload[132:136] = len(address).to_bytes(4, "little") + payload[136 : 136 + len(address)] = address + session = LiveHistorySession("192.0.2.10") + + session.process_packet( + UdpPacket( + 1.0, + "198.51.100.20", + "192.0.2.10", + 30000, + 40000, + bytes(payload), + protocol="tcp", + ) + ) + + self.assertEqual(session.server_id, "23004") + + @patch("builtins.input", side_effect=["9", "3"]) + def test_server_prompt_retries_then_returns_selected_server_id(self, _input): + self.assertEqual(console.prompt_server_id(), "23003") + + @patch("builtins.input", return_value="") + def test_server_prompt_can_be_skipped(self, _input): + self.assertIsNone(console.prompt_server_id()) + + @patch("nte_history_exporter.console.wait_for_keypress") + def test_wait_for_close_waits_for_a_second_keypress(self, wait_for_keypress): + console.wait_for_close() + + wait_for_keypress.assert_called_once_with() + def test_debug_csv_includes_exporter_version(self): with TemporaryDirectory() as tmp: path = Path(tmp) / "debug.csv" From d04a80c3f111b90265b811ba23f93590f332c74d Mon Sep 17 00:00:00 2001 From: Golumpa Date: Thu, 16 Jul 2026 01:40:38 +0100 Subject: [PATCH 8/8] Bump version to 0.2.0 also improved github workflows a bit --- .github/workflows/release.yml | 53 ++++++++++++++++++--------- docs/export-format.md | 2 +- pyproject.toml | 2 +- src/nte_history_exporter/__init__.py | 2 +- src/nte_history_exporter/constants.py | 2 +- 5 files changed, 39 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f80552c..d88fecc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,6 +6,10 @@ on: permissions: contents: write +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: true + jobs: validate-release: name: Validate Release @@ -22,12 +26,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | version="$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")" - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - tag="v${version}" - else - safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')" - tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}" + if [ "${{ github.ref }}" != "refs/heads/main" ]; then + echo "Development release dev-v${version} will be created or updated." + exit 0 fi + tag="v${version}" if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow." @@ -167,8 +170,7 @@ jobs: if [ "${{ github.ref }}" = "refs/heads/main" ]; then tag="v${version}" else - safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')" - tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}" + tag="dev-v${version}" fi repo_url="${{ github.server_url }}/${{ github.repository }}" previous_tag="$(git tag --merged HEAD --sort=-v:refname | grep -E '^v[0-9]+(\.[0-9]+)*$' | grep -vx "$tag" | head -n 1 || true)" @@ -223,11 +225,10 @@ jobs: version="${{ steps.package-version.outputs.version }}" if [ "${{ github.ref }}" = "refs/heads/main" ]; then tag="v${version}" - prerelease_args=() + title="v${version} Release" else - safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')" - tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}" - prerelease_args=(--prerelease) + tag="dev-v${version}" + title="v${version} Development Build" fi files=( nte-history-exporter.exe @@ -238,14 +239,30 @@ jobs: nte-history-exporter-v${version}-x86_64-apple-darwin.zip ) - if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then - echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow." - exit 1 + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then + echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow." + exit 1 + fi + + if gh release view "$tag" >/dev/null 2>&1; then + echo "::error title=GitHub release already exists::${tag} already has a GitHub release. Bump project.version in pyproject.toml before running the release workflow." + exit 1 + fi + + gh release create "$tag" "${files[@]}" --title "$title" --notes-file release-notes.md + exit 0 fi + # Development releases are intentionally mutable. Keep one prerelease + # per package version and move it to the commit that produced the + # latest successful build. + git tag --force "$tag" "$GITHUB_SHA" + git push origin "refs/tags/${tag}" --force + if gh release view "$tag" >/dev/null 2>&1; then - echo "::error title=GitHub release already exists::${tag} already has a GitHub release. Bump project.version in pyproject.toml before running the release workflow." - exit 1 + gh release edit "$tag" --title "$title" --notes-file release-notes.md --prerelease + gh release upload "$tag" "${files[@]}" --clobber + else + gh release create "$tag" "${files[@]}" --title "$title" --notes-file release-notes.md --prerelease fi - - gh release create "$tag" "${files[@]}" --title "${tag} Release" --notes-file release-notes.md "${prerelease_args[@]}" diff --git a/docs/export-format.md b/docs/export-format.md index 1b9ea37..863442f 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -15,7 +15,7 @@ decoder-only offsets. "capture_source": "npcap", "exporter": { "name": "nte-history-exporter", - "version": "0.1.12" + "version": "0.2.0" }, "banner": { "id": "Lottery_Permanent", diff --git a/pyproject.toml b/pyproject.toml index f87550c..39e30a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nte-history-exporter" -version = "0.1.12" +version = "0.2.0" description = "Cross-platform Neverness to Everness pull-history exporter." requires-python = ">=3.10" dependencies = [] diff --git a/src/nte_history_exporter/__init__.py b/src/nte_history_exporter/__init__.py index 74acd0e..d3ec452 100644 --- a/src/nte_history_exporter/__init__.py +++ b/src/nte_history_exporter/__init__.py @@ -1 +1 @@ -__version__ = "0.1.12" +__version__ = "0.2.0" diff --git a/src/nte_history_exporter/constants.py b/src/nte_history_exporter/constants.py index 0491f89..f5afd5b 100644 --- a/src/nte_history_exporter/constants.py +++ b/src/nte_history_exporter/constants.py @@ -13,7 +13,7 @@ ARC_BANNER_NAME = "Arc Miracle Box" MYSTERY_BOX_BANNER_ID = "Gashapon_MysteryBox" MYSTERY_BOX_BANNER_NAME = "Mystery Box" EXPORTER_NAME = "nte-history-exporter" -EXPORTER_VERSION = "0.1.12" +EXPORTER_VERSION = "0.2.0" HISTORY_REQUEST_BANNER = 4220 HISTORY_REQUEST_LENGTH = 45