from __future__ import annotations import csv import unittest from dataclasses import replace from pathlib import Path from tempfile import TemporaryDirectory from unittest.mock import patch from nte_history_exporter.decoder.arc import ( build_arc_rows_from_pairs, make_arc_uid, parse_arc_response, ) from nte_history_exporter.decoder.boundary import annotate_groups, make_uid from nte_history_exporter.decoder.protocol import decode_response_records from nte_history_exporter.decoder.structured_protocol import ( FORK_BLOCK_KIND, FORK_MARKER, MONOPOLY_BLOCK_KIND, MONOPOLY_ENVELOPE_FOOTER, MONOPOLY_MARKER, PROTOCOL_CONSTANT, ProtocolEnvelope, StructuredProtocolAssembler, parse_structured_blocks, parse_structured_records, ) from nte_history_exporter.decoder.run import build_rows_from_pairs from nte_history_exporter.export.csv_export import write_csv from nte_history_exporter.export.json_export import build_export_json from tests.support import fixture_payload STRUCTURED_TICKS = 639_131_653_353_040_000 def fstring(value: str) -> bytes: raw = value.encode("utf-8") + b"\0" return len(raw).to_bytes(4, "little") + raw def monopoly_payload( item_spec: str, *, ticks: int = STRUCTURED_TICKS, roll_points: int = 2, secondary_item_id: str = "", secondary_count: int = 0, pool_id: str = "CardPool_Character", ) -> bytes: row = ( roll_points.to_bytes(4, "little") + fstring(item_spec) + (0).to_bytes(4, "little") + secondary_count.to_bytes(4, "little") + fstring(secondary_item_id) + fstring(item_spec.split(",", 1)[0]) + fstring(pool_id) + ticks.to_bytes(8, "little") ) return ( MONOPOLY_MARKER + b"\0" + (0).to_bytes(4, "little") + len(row).to_bytes(4, "little") + (1).to_bytes(4, "little") + row ) def fork_payload(item_spec: str, *, ticks: int = STRUCTURED_TICKS) -> bytes: row = fstring(item_spec) + fstring("ForkLottery_AnHunQu") + ticks.to_bytes(8, "little") return ( FORK_MARKER + b"\0" + (0).to_bytes(4, "little") + len(row).to_bytes(4, "little") + (1).to_bytes(4, "little") + row ) def bit_pack_after_eight_byte_header(payload: bytes, shift: int) -> bytes: packed = int.from_bytes(payload, "little") << shift return bytes(8) + packed.to_bytes(len(payload) + 1, "little") def enveloped_monopoly_payload( item_spec: str, *, page_index: int, query_high: bool, pool_token: int = 256, shift: int = 3, ) -> bytes: envelope = ( PROTOCOL_CONSTANT.to_bytes(4, "little") + ((0x80000000 if query_high else 0)).to_bytes(4, "little") + page_index.to_bytes(4, "little") + MONOPOLY_BLOCK_KIND.to_bytes(4, "little") + pool_token.to_bytes(4, "little") + MONOPOLY_ENVELOPE_FOOTER.to_bytes(4, "little") + b"\0\0" ) return bit_pack_after_eight_byte_header(envelope + monopoly_payload(item_spec), shift) def enveloped_fork_payload( item_spec: str, *, page_index: int, query_high: bool, shift: int = 3, ) -> bytes: envelope = ( PROTOCOL_CONSTANT.to_bytes(4, "little") + ((0x80000000 if query_high else 0)).to_bytes(4, "little") + page_index.to_bytes(4, "little") + FORK_BLOCK_KIND.to_bytes(4, "little") + b"\0" ) return bit_pack_after_eight_byte_header(envelope + fork_payload(item_spec), shift) def assembled_block(item_id: str, segment_index: int, source_index: int = 0): block = parse_structured_blocks( monopoly_payload(item_id), "monopoly", source_index=source_index )[0] envelope = ProtocolEnvelope( record_type="monopoly", stream_key="monopoly:256", page_index=segment_index // 2, query_high=segment_index % 2 == 0, segment_index=segment_index, ) return replace(block, envelope=envelope) class StructuredProtocolTests(unittest.TestCase): def test_structured_monopoly_parser_enriches_the_existing_decoder(self): payload = monopoly_payload( "Fashion_vehicle_1010_V008,3", secondary_item_id="Dice_ticket_02", secondary_count=5, ) rows = decode_response_records(payload) self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["decoder_mode"], "heuristic_enriched") self.assertEqual(rows[0]["reward_id"], "Fashion_vehicle_1010_V008") self.assertEqual(rows[0]["reward_name"], "Tiger Incoming! - Livery") self.assertEqual(rows[0]["quantity"], 3) self.assertEqual(rows[0]["dice"], 2) self.assertEqual(rows[0]["secondary_reward_id"], "Dice_ticket_02") self.assertEqual(rows[0]["secondary_quantity"], 5) self.assertEqual(rows[0]["structured_pool_id"], "CardPool_Character") def test_structured_reward_mapping_is_case_insensitive_and_canonical(self): row = decode_response_records(monopoly_payload("DICENORMAL,1"))[0] self.assertEqual(row["reward_id"], "DiceNormal") self.assertEqual(row["reward_name"], "Fabricated Dice") def test_structured_monopoly_parser_falls_back_when_heuristic_returns_no_rows(self): payload = monopoly_payload("Dice_ticket_02,50", roll_points=0) with patch("nte_history_exporter.decoder.protocol._decode_aligned_response_records", return_value=[]): rows = decode_response_records(payload) self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["decoder_mode"], "structured_fallback") self.assertEqual(rows[0]["reward_id"], "Dice_ticket_02") self.assertEqual(rows[0]["quantity"], 50) self.assertEqual(rows[0]["result_type"], "points_gift") def test_structured_fork_parser_is_a_complete_fallback(self): rows = parse_arc_response(fork_payload("fork_dustbin,2")) self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["decoder_mode"], "structured_fallback") self.assertEqual(rows[0]["reward_id"], "fork_dustbin") self.assertEqual(rows[0]["reward_name"], "Dangerous Game") self.assertEqual(rows[0]["structured_pool_id"], "ForkLottery_AnHunQu") def test_structured_parser_realigns_bit_packed_payload(self): payload = bit_pack_after_eight_byte_header(monopoly_payload("1003,1"), 3) rows = parse_structured_records(payload, "monopoly") self.assertEqual(len(rows), 1) self.assertEqual(rows[0].item_id, "1003") self.assertEqual(rows[0].protocol_view, "shift8:3") def test_shifted_parser_allows_missing_declared_protocol_padding(self): cases = ( ("monopoly", monopoly_payload("1003,1"), 3, "1003"), ("fork", fork_payload("fork_dustbin"), 2, "fork_dustbin"), ) for record_type, raw_payload, shift, expected_item_id in cases: with self.subTest(record_type=record_type): payload_with_declared_padding = bytearray(raw_payload) marker = MONOPOLY_MARKER if record_type == "monopoly" else FORK_MARKER declared_size_pos = len(marker) + 1 + 4 declared_size = int.from_bytes( payload_with_declared_padding[declared_size_pos : declared_size_pos + 4], "little", ) payload_with_declared_padding[ declared_size_pos : declared_size_pos + 4 ] = (declared_size + 3).to_bytes(4, "little") packed = bit_pack_after_eight_byte_header( bytes(payload_with_declared_padding), shift ) rows = parse_structured_records(packed, record_type) self.assertEqual([row.item_id for row in rows], [expected_item_id]) def test_protocol_envelope_exposes_stream_and_segment_identity(self): payload = enveloped_monopoly_payload("1003", page_index=2, query_high=False) block = parse_structured_blocks(payload, "monopoly")[0] self.assertIsNotNone(block.envelope) self.assertEqual(block.envelope.stream_key, "monopoly:256") self.assertEqual(block.envelope.page_index, 2) self.assertFalse(block.envelope.query_high) self.assertEqual(block.envelope.segment_index, 3) def test_assembler_orders_segments_and_ignores_retransmissions(self): assembler = StructuredProtocolAssembler() segment_one = assembled_block("1010", 1) assembler.add_blocks([segment_one, assembled_block("1003", 0), segment_one]) rows = assembler.rows("monopoly") self.assertEqual([row.item_id for row in rows], ["1003", "1010"]) self.assertEqual([row.generation_index for row in rows], [0, 0]) self.assertEqual(assembler.warnings, []) def test_assembler_never_deduplicates_blocks_without_envelopes(self): assembler = StructuredProtocolAssembler() block = parse_structured_blocks(monopoly_payload("1003"), "monopoly")[0] assembler.add_blocks([block, block]) self.assertEqual([row.item_id for row in assembler.rows("monopoly")], ["1003", "1003"]) def test_new_complete_generation_replaces_old_snapshot(self): assembler = StructuredProtocolAssembler() assembler.add_blocks([assembled_block("1003", 0), assembled_block("1010", 1)]) assembler.add_blocks([assembled_block("1020", 0), assembled_block("1021", 1)]) rows = assembler.rows("monopoly") self.assertEqual([row.item_id for row in rows], ["1020", "1021"]) self.assertEqual([row.generation_index for row in rows], [1, 1]) def test_partial_generation_merges_only_on_unique_overlap(self): assembler = StructuredProtocolAssembler() assembler.add_blocks( [ assembled_block("1003", 0), assembled_block("1010", 1), assembled_block("1020", 2), ] ) assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)]) rows = assembler.rows("monopoly") self.assertEqual([row.item_id for row in rows], ["1099", "1010", "1020"]) self.assertEqual(assembler.warnings, []) def test_ambiguous_partial_generation_keeps_proven_snapshot(self): assembler = StructuredProtocolAssembler() assembler.add_blocks( [ assembled_block("1003", 0), assembled_block("1010", 1), assembled_block("1003", 2), assembled_block("1010", 3), ] ) assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)]) rows = assembler.rows("monopoly") self.assertEqual([row.item_id for row in rows], ["1003", "1010", "1003", "1010"]) self.assertEqual(assembler.warnings[0]["code"], "AMBIGUOUS_STRUCTURED_SNAPSHOT") def test_pair_assembly_is_used_only_for_all_structured_fallback(self): segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False) segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True) pairs = [ (2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"), (1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"), ] with patch("nte_history_exporter.decoder.protocol._decode_aligned_response_records", return_value=[]): rows = build_rows_from_pairs(pairs) annotated = annotate_groups(rows) self.assertEqual([row["reward_id"] for row in annotated], ["1003", "1010"]) self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in annotated)) self.assertEqual(annotated[0]["uid"], make_uid(annotated[0], 0)) def test_pair_assembly_cannot_reorder_successful_heuristic_rows(self): segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False) segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True) pairs = [ (2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"), (1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"), ] rows = build_rows_from_pairs(pairs) self.assertEqual([row["reward_id"] for row in rows], ["1010", "1003"]) self.assertTrue(all(row["decoder_mode"] == "heuristic_enriched" for row in rows)) self.assertTrue(all("structured_assembly" not in row for row in rows)) def test_arc_fallback_uses_the_same_segment_assembly_and_uid_order(self): segment_one = enveloped_fork_payload("fork_vine", page_index=1, query_high=False) segment_zero = enveloped_fork_payload("fork_dustbin", page_index=0, query_high=True) pairs = [ (2, 4, 1, 1.0, 2, 1.1, segment_one, "arc_miracle_box"), (1, 2, 3, 1.2, 4, 1.3, segment_zero, "arc_miracle_box"), ] rows = build_arc_rows_from_pairs(pairs) self.assertEqual([row["reward_id"] for row in rows], ["fork_dustbin", "fork_vine"]) self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in rows)) self.assertEqual( rows[0]["uid"], make_arc_uid(rows[0]["timestamp_raw_hex"], rows[0]["timestamp_group_ordinal"]), ) def test_matching_structured_data_enriches_without_replacing_heuristic_identity(self): heuristic_payload = fixture_payload("limited-points-gift-1") original = decode_response_records(heuristic_payload)[0] structured_ticks = original["timestamp_ticks"] // 4 combined = heuristic_payload + monopoly_payload( "1020,7", ticks=structured_ticks, roll_points=0, pool_id="CardPool_Character", ) enriched = decode_response_records(combined)[0] self.assertEqual(enriched["decoder_mode"], "heuristic_enriched") self.assertEqual(enriched["quantity"], 7) self.assertEqual(enriched["reward_id"], original["reward_id"]) self.assertEqual(enriched["timestamp_raw_hex"], original["timestamp_raw_hex"]) self.assertEqual(enriched["timestamp_ticks"], original["timestamp_ticks"]) def test_conflicting_structured_data_cannot_override_heuristic_record(self): heuristic_payload = fixture_payload("limited-points-gift-1") original = decode_response_records(heuristic_payload)[0] combined = heuristic_payload + monopoly_payload( "1003,99", ticks=original["timestamp_ticks"] // 4, roll_points=6, ) decoded = decode_response_records(combined)[0] self.assertNotIn("decoder_mode", decoded) self.assertEqual(decoded["reward_id"], original["reward_id"]) self.assertEqual(decoded["quantity"], original["quantity"]) self.assertEqual(decoded["dice"], original["dice"]) def test_malformed_structured_block_fails_closed(self): malformed = ( MONOPOLY_MARKER + b"\0" + (0).to_bytes(4, "little") + (9999).to_bytes(4, "little") + (1).to_bytes(4, "little") ) self.assertEqual(decode_response_records(malformed), []) def test_structured_diagnostics_are_debug_only(self): row = decode_response_records(monopoly_payload("1003,1"))[0] row.update( { "uid": "stable-test-uid", "export_record": True, "pool_group_id": "Lottery_Permanent", "timestamp_group_ordinal": 0, } ) export_record = build_export_json([row], [])["records"][0] self.assertNotIn("decoder_mode", export_record) self.assertNotIn("structured_pool_id", export_record) self.assertNotIn("secondary_reward_id", export_record) with TemporaryDirectory() as tmp: path = Path(tmp) / "debug.csv" write_csv(path, [row]) with path.open(newline="", encoding="utf-8") as handle: debug_record = next(csv.DictReader(handle)) self.assertEqual(debug_record["decoder_mode"], "heuristic_enriched") self.assertEqual(debug_record["structured_pool_id"], "CardPool_Character") if __name__ == "__main__": unittest.main()