Stabilize UID generation and page-first dice parsing

docs/export-format.md

@@ -92,18 +92,22 @@ nice names or ranks.
 
 ## UID generation
 
-The `uid` is the first 32 hex characters of `sha256(source)`. We generate our own roll UID as the game does not send their own, so to make things trackable and to help prevent duplicates we create our own UID with a selection of feilds making each entry in the history 100% unique. The way it is done means on a rescan the same UID is generated for the same history item even if it is further in the history and you have pulled more since the last scan.
+The `uid` is the first 32 hex characters of `sha256(source)`. The game does not
+appear to send a stable pull/reward row ID, so the exporter builds one from the
+history pool, raw packet timestamp, and ordinal inside that timestamp group.
+Decoded content such as dice result, reward ID, and quantity is intentionally
+excluded so decoder fixes do not change the identity of an already-captured row.
 
 Monopoly source:
 
 ```text
-nte|monopoly|pool_group_id|timestamp_raw|timestamp_group_ordinal|roll_result|reward_key_hex|quantity
+nte|monopoly|pool_group_id|timestamp_raw|timestamp_group_ordinal
 ```
 
 Arc source:
 
 ```text
-nte|gashapon|pool_group_id|timestamp_raw|timestamp_group_ordinal|reward_key_hex
+nte|gashapon|pool_group_id|timestamp_raw|timestamp_group_ordinal
 ```
 
 ## Example records

docs/packet-format.md

@@ -25,6 +25,9 @@ Decoded fields:
 
 - `roll_result = first u32 / 4`
 - `roll_result = 0` means Points Gift
+- Some page-first records include a one-byte page prefix and an extra `0x14`
+  field before the real dice u32. In that shape, the real dice u32 is at offset
+  9 within the record chunk, not the earlier `0x14` field.
 - Some page-first records have a short prefix before the record body. For these, a hidden signed source flag immediately after the visible dice field overrides the visible dice:
   - `source_flag = 0` means Points Gift
   - `source_flag = -4` means Chase Reward

src/nte_history_exporter/decoder/arc.py

@@ -146,8 +146,8 @@ def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
     return rows
 
 
-def make_arc_uid(timestamp_raw: str, ordinal: int, arc_key_hex: str) -> str:
+def make_arc_uid(timestamp_raw: str, ordinal: int) -> str:
-    source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal), arc_key_hex])
+    source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal)])
     return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32]
 
 
@@ -165,7 +165,7 @@ def annotate_arc_groups(rows: list[dict[str, Any]]) -> None:
             row["timestamp_group_index"] = group_index
             row["timestamp_group_ordinal"] = ordinal
             row["timestamp_group_size_seen"] = len(indexes)
-            row["uid"] = make_arc_uid(timestamp_raw, ordinal, row["reward_key_hex"])
+            row["uid"] = make_arc_uid(timestamp_raw, ordinal)
             row["uid_status"] = "stable"
             row["export_record"] = True
             row["skip_reason"] = ""

src/nte_history_exporter/decoder/boundary.py

@@ -14,9 +14,6 @@ def make_uid(record: dict[str, Any], ordinal: int) -> str:
             str(record.get("pool_group_id", BANNER_ID)),
             str(record.get("timestamp_raw_hex", "")),
             str(ordinal),
-            str(record.get("dice", "")),
-            str(record.get("reward_key_hex", "")),
-            str(record.get("quantity", "")),
         ]
     )
     return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32]

src/nte_history_exporter/decoder/protocol.py

@@ -132,7 +132,20 @@ def extract_key(chunk_without_marker: bytes) -> str:
     return "" if best is None else chunk_without_marker[best:].hex()
 
 
+def _page_first_prefixed_dice_raw(chunk_without_marker: bytes) -> int | None:
+    if len(chunk_without_marker) >= 17 and chunk_without_marker[0] == 0:
+        prefix_field = struct.unpack_from("<I", chunk_without_marker, 5)[0]
+        prefixed_dice_raw = struct.unpack_from("<I", chunk_without_marker, 9)[0]
+        if prefix_field == 20 and prefixed_dice_raw in VALID_DICE_FIELDS:
+            return prefixed_dice_raw
+    return None
+
+
 def extract_dice(chunk_without_marker: bytes) -> tuple[int | None, int | None, int | None]:
+    prefixed_dice_raw = _page_first_prefixed_dice_raw(chunk_without_marker)
+    if prefixed_dice_raw is not None:
+        return (0 if prefixed_dice_raw == 0 else prefixed_dice_raw // 4), prefixed_dice_raw, 9
+
     for off in range(0, min(16, max(0, len(chunk_without_marker) - 3))):
         val = struct.unpack_from("<I", chunk_without_marker, off)[0]
         if val in VALID_DICE_FIELDS:
@@ -201,16 +214,23 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str,
         chunk = response_content[prev:marker_offset]
         full_record = response_content[prev : marker_offset + len(marker) + 8]
         dice, dice_raw, dice_offset = extract_dice(chunk)
-        if dice is None and len(chunk) > 32:
-            embedded_candidates = []
         original_key = extract_key(chunk)
         original_key_bytes = bytes.fromhex(original_key) if original_key else b""
+        key_position = chunk.find(original_key_bytes) if original_key_bytes else len(chunk)
+        should_try_embedded_trim = (
+            dice is None
+            or (_page_first_prefixed_dice_raw(chunk) is None and key_position > 32)
+        )
+        if should_try_embedded_trim and len(chunk) > 32:
+            embedded_candidates = []
             for trim in range(1, min(96, len(chunk))):
                 candidate = chunk[trim:]
                 candidate_dice, candidate_raw, candidate_offset = extract_dice(candidate)
+                candidate_is_page_first = _page_first_prefixed_dice_raw(candidate) is not None
                 if (
                     candidate_dice is not None
-                    and candidate_offset in (0, 5)
+                    and candidate_offset in (0, 5, 9)
+                    and (dice is None or candidate_is_page_first)
                     and extract_key(candidate) == original_key
                 ):
                     key_count = candidate.count(original_key_bytes) if original_key_bytes else 0
@@ -218,6 +238,7 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str,
                     embedded_candidates.append(
                         (
                             -key_count,
+                            not candidate_is_page_first,
                             candidate_offset != 5,
                             key_position,
                             -trim,
@@ -229,7 +250,7 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str,
                         )
                     )
             if embedded_candidates:
-                _, _, _, _, trim, chunk, dice, dice_raw, dice_offset = min(embedded_candidates)
+                _, _, _, _, _, trim, chunk, dice, dice_raw, dice_offset = min(embedded_candidates)
                 record_start = prev + trim
                 full_record = response_content[prev + trim : marker_offset + len(marker) + 8]
         key_hex = extract_key(chunk)

tests/test_boundary_export.py

@@ -300,9 +300,9 @@ class BoundaryExportTests(unittest.TestCase):
     def test_uid_source_matches_v4_reference(self):
         rows = load_reference_csv("monopoly_history_poc_10_all_44_pages_v4.csv")
         first = rows[0]
-        self.assertEqual(make_uid(first, 0), "5adcf52282e15445466863b271f3b745")
+        self.assertEqual(make_uid(first, 0), "f2c72f0a80b79216bf15661521620693")
 
-    def test_uid_uses_detected_pool_group_id(self):
+    def test_uid_uses_pool_timestamp_and_ordinal_only(self):
         row = {
             "pool_group_id": "Lottery_LimitedCharacter",
             "timestamp_raw_hex": "40e93247c3097b23",
@@ -310,7 +310,18 @@ class BoundaryExportTests(unittest.TestCase):
             "reward_key_hex": "10a58d957dd1a58dad95d17dc1c800",
             "quantity": 50,
         }
-        self.assertNotEqual(make_uid(row, 0), "5adcf52282e15445466863b271f3b745")
+        changed_content = {
+            **row,
+            "dice": 1,
+            "reward_key_hex": "98bdc9ad7dd9a5b99501",
+            "quantity": 1,
+        }
+        changed_pool = {**row, "pool_group_id": "Lottery_Permanent"}
+
+        self.assertEqual(make_uid(row, 0), "74a9ef4aacde549dfe8e8e7cc6ddd65b")
+        self.assertEqual(make_uid(changed_content, 0), make_uid(row, 0))
+        self.assertNotEqual(make_uid(changed_pool, 0), make_uid(row, 0))
+        self.assertNotEqual(make_uid(row, 1), make_uid(row, 0))
 
     def test_pages_1_to_5_exports_every_row(self):
         rows = load_reference_csv("monopoly_history_poc_13_pages_1_to_5_v4.csv")
@@ -592,7 +603,7 @@ class BoundaryExportTests(unittest.TestCase):
         self.assertEqual(decoded["dice_raw_u32"], -4)
         self.assertEqual(decoded["reward_id"], "Dice_ticket_01")
         self.assertEqual(decoded["quantity"], 30)
-        self.assertEqual(make_uid(decoded, int(reference["timestamp_group_ordinal"])), "7d035ec098f856f81b403ea538810145")
+        self.assertEqual(make_uid(decoded, int(reference["timestamp_group_ordinal"])), "23dc293f39f18e94a81ccaaf7e1a67eb")
 
     def test_warp_piece_chase_subrecord_without_prefix_marker_is_chase_reward(self):
         decoded = decode_single_record(
@@ -608,6 +619,32 @@ class BoundaryExportTests(unittest.TestCase):
         self.assertEqual(decoded["reward_name"], "Warp Piece")
         self.assertEqual(decoded["quantity"], 30)
 
+    def test_page_first_prefix_uses_real_dice_field(self):
+        cases = [
+            (
+                "003006000014000000040000002800000098bdc9ad7dd9a5b995010000000008000000"
+                "3c00000010a58d957dd1a58dad95d17dc1c4002800000098bdc9ad7dd9a5b995014c"
+                "0000000c85c99141bdbdb17d0da185c9858dd195c901c0dd53bd2b137b23",
+                1,
+                "fork_vine",
+            ),
+            (
+                "00c8060000140000001000000014000000c4c0d4d400000000000400000014000000"
+                "c4c0d4d400440000000c85c99141bdbdb17d3995dd49bdb1950100d929e115087b23",
+                4,
+                "1055",
+            ),
+        ]
+        for record_hex, expected_dice, reward_id in cases:
+            with self.subTest(reward_id=reward_id):
+                decoded = decode_single_record(record_hex)
+
+                self.assertEqual(decoded["dice"], expected_dice)
+                self.assertEqual(decoded["dice_raw_u32"], expected_dice * 4)
+                self.assertEqual(decoded["dice_offset_in_record"], 9)
+                self.assertEqual(decoded["result_type"], "dice")
+                self.assertEqual(decoded["reward_id"], reward_id)
+
     def test_batched_monopoly_response_normalizes_embedded_page_header(self):
         page_7 = [load_v7_row("limited_all_04_v7.csv", row) for row in range(31, 36)]
         page_8 = [load_v7_row("limited_all_04_v7.csv", row) for row in range(36, 41)]
@@ -626,7 +663,7 @@ class BoundaryExportTests(unittest.TestCase):
 
         self.assertEqual(len(decoded), 10)
         self.assertEqual(decoded[5]["reward_id"], page_8[0]["reward_id"])
-        self.assertEqual(decoded[5]["dice"], 5)
+        self.assertEqual(decoded[5]["dice"], 4)
         self.assertEqual(decoded[5]["result_type"], "dice")
         self.assertEqual(decoded[5]["record_hex"], page_8[0]["record_hex"])
 
@@ -665,10 +702,7 @@ class BoundaryExportTests(unittest.TestCase):
         self.assertEqual(decode_arc_key(bytes.fromhex(row["arc_key_hex"])), "fork_nonos")
         _ticks, _unix, decoded = decode_arc_timestamp(bytes.fromhex(row["timestamp_raw_hex"]))
         self.assertEqual(decoded, "2026-06-10 23:46:29")
-        self.assertEqual(
+        self.assertEqual(make_arc_uid(row["timestamp_raw_hex"], int(row["timestamp_group_ordinal"])), "4435d9729fa8fd0eaf1b1ad7aa4d2172")
-            make_arc_uid(row["timestamp_raw_hex"], int(row["timestamp_group_ordinal"]), row["arc_key_hex"]),
-            row["uid"],
-        )
 
     def test_arc_response_parser_matches_reference_first_page(self):
         reference_rows = load_arc_csv("arc_pull_10_all_pages_v2.csv")[:5]