Add safe snapshot and segment assembly
Add envelope-aware snapshot and segment assembly for structured Monopoly and Arc history records.
This commit is contained in:
Golumpa
parent
a5312520e7
commit
73bb100cd6
8 changed files with 601 additions and 23 deletions
|
|
@ -2,17 +2,31 @@ from __future__ import annotations
|
|||
|
||||
import csv
|
||||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
from nte_history_exporter.decoder.arc import parse_arc_response
|
||||
from nte_history_exporter.decoder.arc import (
|
||||
build_arc_rows_from_pairs,
|
||||
make_arc_uid,
|
||||
parse_arc_response,
|
||||
)
|
||||
from nte_history_exporter.decoder.boundary import annotate_groups, make_uid
|
||||
from nte_history_exporter.decoder.protocol import decode_response_records
|
||||
from nte_history_exporter.decoder.structured_protocol import (
|
||||
FORK_BLOCK_KIND,
|
||||
FORK_MARKER,
|
||||
MONOPOLY_BLOCK_KIND,
|
||||
MONOPOLY_ENVELOPE_FOOTER,
|
||||
MONOPOLY_MARKER,
|
||||
PROTOCOL_CONSTANT,
|
||||
ProtocolEnvelope,
|
||||
StructuredProtocolAssembler,
|
||||
parse_structured_blocks,
|
||||
parse_structured_records,
|
||||
)
|
||||
from nte_history_exporter.decoder.run import build_rows_from_pairs
|
||||
from nte_history_exporter.export.csv_export import write_csv
|
||||
from nte_history_exporter.export.json_export import build_export_json
|
||||
from tests.support import fixture_payload
|
||||
|
|
@ -72,6 +86,57 @@ def bit_pack_after_eight_byte_header(payload: bytes, shift: int) -> bytes:
|
|||
return bytes(8) + packed.to_bytes(len(payload) + 1, "little")
|
||||
|
||||
|
||||
def enveloped_monopoly_payload(
|
||||
item_spec: str,
|
||||
*,
|
||||
page_index: int,
|
||||
query_high: bool,
|
||||
pool_token: int = 256,
|
||||
shift: int = 3,
|
||||
) -> bytes:
|
||||
envelope = (
|
||||
PROTOCOL_CONSTANT.to_bytes(4, "little")
|
||||
+ ((0x80000000 if query_high else 0)).to_bytes(4, "little")
|
||||
+ page_index.to_bytes(4, "little")
|
||||
+ MONOPOLY_BLOCK_KIND.to_bytes(4, "little")
|
||||
+ pool_token.to_bytes(4, "little")
|
||||
+ MONOPOLY_ENVELOPE_FOOTER.to_bytes(4, "little")
|
||||
+ b"\0\0"
|
||||
)
|
||||
return bit_pack_after_eight_byte_header(envelope + monopoly_payload(item_spec), shift)
|
||||
|
||||
|
||||
def enveloped_fork_payload(
|
||||
item_spec: str,
|
||||
*,
|
||||
page_index: int,
|
||||
query_high: bool,
|
||||
shift: int = 3,
|
||||
) -> bytes:
|
||||
envelope = (
|
||||
PROTOCOL_CONSTANT.to_bytes(4, "little")
|
||||
+ ((0x80000000 if query_high else 0)).to_bytes(4, "little")
|
||||
+ page_index.to_bytes(4, "little")
|
||||
+ FORK_BLOCK_KIND.to_bytes(4, "little")
|
||||
+ b"\0"
|
||||
)
|
||||
return bit_pack_after_eight_byte_header(envelope + fork_payload(item_spec), shift)
|
||||
|
||||
|
||||
def assembled_block(item_id: str, segment_index: int, source_index: int = 0):
|
||||
block = parse_structured_blocks(
|
||||
monopoly_payload(item_id), "monopoly", source_index=source_index
|
||||
)[0]
|
||||
envelope = ProtocolEnvelope(
|
||||
record_type="monopoly",
|
||||
stream_key="monopoly:256",
|
||||
page_index=segment_index // 2,
|
||||
query_high=segment_index % 2 == 0,
|
||||
segment_index=segment_index,
|
||||
)
|
||||
return replace(block, envelope=envelope)
|
||||
|
||||
|
||||
class StructuredProtocolTests(unittest.TestCase):
|
||||
def test_structured_monopoly_parser_enriches_the_existing_decoder(self):
|
||||
payload = monopoly_payload(
|
||||
|
|
@ -122,6 +187,126 @@ class StructuredProtocolTests(unittest.TestCase):
|
|||
self.assertEqual(rows[0].item_id, "1003")
|
||||
self.assertEqual(rows[0].protocol_view, "shift8:3")
|
||||
|
||||
def test_protocol_envelope_exposes_stream_and_segment_identity(self):
|
||||
payload = enveloped_monopoly_payload("1003", page_index=2, query_high=False)
|
||||
|
||||
block = parse_structured_blocks(payload, "monopoly")[0]
|
||||
|
||||
self.assertIsNotNone(block.envelope)
|
||||
self.assertEqual(block.envelope.stream_key, "monopoly:256")
|
||||
self.assertEqual(block.envelope.page_index, 2)
|
||||
self.assertFalse(block.envelope.query_high)
|
||||
self.assertEqual(block.envelope.segment_index, 3)
|
||||
|
||||
def test_assembler_orders_segments_and_ignores_retransmissions(self):
|
||||
assembler = StructuredProtocolAssembler()
|
||||
segment_one = assembled_block("1010", 1)
|
||||
assembler.add_blocks([segment_one, assembled_block("1003", 0), segment_one])
|
||||
|
||||
rows = assembler.rows("monopoly")
|
||||
|
||||
self.assertEqual([row.item_id for row in rows], ["1003", "1010"])
|
||||
self.assertEqual([row.generation_index for row in rows], [0, 0])
|
||||
self.assertEqual(assembler.warnings, [])
|
||||
|
||||
def test_assembler_never_deduplicates_blocks_without_envelopes(self):
|
||||
assembler = StructuredProtocolAssembler()
|
||||
block = parse_structured_blocks(monopoly_payload("1003"), "monopoly")[0]
|
||||
|
||||
assembler.add_blocks([block, block])
|
||||
|
||||
self.assertEqual([row.item_id for row in assembler.rows("monopoly")], ["1003", "1003"])
|
||||
|
||||
def test_new_complete_generation_replaces_old_snapshot(self):
|
||||
assembler = StructuredProtocolAssembler()
|
||||
assembler.add_blocks([assembled_block("1003", 0), assembled_block("1010", 1)])
|
||||
assembler.add_blocks([assembled_block("1020", 0), assembled_block("1021", 1)])
|
||||
|
||||
rows = assembler.rows("monopoly")
|
||||
|
||||
self.assertEqual([row.item_id for row in rows], ["1020", "1021"])
|
||||
self.assertEqual([row.generation_index for row in rows], [1, 1])
|
||||
|
||||
def test_partial_generation_merges_only_on_unique_overlap(self):
|
||||
assembler = StructuredProtocolAssembler()
|
||||
assembler.add_blocks(
|
||||
[
|
||||
assembled_block("1003", 0),
|
||||
assembled_block("1010", 1),
|
||||
assembled_block("1020", 2),
|
||||
]
|
||||
)
|
||||
assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)])
|
||||
|
||||
rows = assembler.rows("monopoly")
|
||||
|
||||
self.assertEqual([row.item_id for row in rows], ["1099", "1010", "1020"])
|
||||
self.assertEqual(assembler.warnings, [])
|
||||
|
||||
def test_ambiguous_partial_generation_keeps_proven_snapshot(self):
|
||||
assembler = StructuredProtocolAssembler()
|
||||
assembler.add_blocks(
|
||||
[
|
||||
assembled_block("1003", 0),
|
||||
assembled_block("1010", 1),
|
||||
assembled_block("1003", 2),
|
||||
assembled_block("1010", 3),
|
||||
]
|
||||
)
|
||||
assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)])
|
||||
|
||||
rows = assembler.rows("monopoly")
|
||||
|
||||
self.assertEqual([row.item_id for row in rows], ["1003", "1010", "1003", "1010"])
|
||||
self.assertEqual(assembler.warnings[0]["code"], "AMBIGUOUS_STRUCTURED_SNAPSHOT")
|
||||
|
||||
def test_pair_assembly_is_used_only_for_all_structured_fallback(self):
|
||||
segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False)
|
||||
segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True)
|
||||
pairs = [
|
||||
(2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"),
|
||||
(1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"),
|
||||
]
|
||||
|
||||
with patch("nte_history_exporter.decoder.protocol._decode_aligned_response_records", return_value=[]):
|
||||
rows = build_rows_from_pairs(pairs)
|
||||
annotated = annotate_groups(rows)
|
||||
|
||||
self.assertEqual([row["reward_id"] for row in annotated], ["1003", "1010"])
|
||||
self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in annotated))
|
||||
self.assertEqual(annotated[0]["uid"], make_uid(annotated[0], 0))
|
||||
|
||||
def test_pair_assembly_cannot_reorder_successful_heuristic_rows(self):
|
||||
segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False)
|
||||
segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True)
|
||||
pairs = [
|
||||
(2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"),
|
||||
(1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"),
|
||||
]
|
||||
|
||||
rows = build_rows_from_pairs(pairs)
|
||||
|
||||
self.assertEqual([row["reward_id"] for row in rows], ["1010", "1003"])
|
||||
self.assertTrue(all(row["decoder_mode"] == "heuristic_enriched" for row in rows))
|
||||
self.assertTrue(all("structured_assembly" not in row for row in rows))
|
||||
|
||||
def test_arc_fallback_uses_the_same_segment_assembly_and_uid_order(self):
|
||||
segment_one = enveloped_fork_payload("fork_vine", page_index=1, query_high=False)
|
||||
segment_zero = enveloped_fork_payload("fork_dustbin", page_index=0, query_high=True)
|
||||
pairs = [
|
||||
(2, 4, 1, 1.0, 2, 1.1, segment_one, "arc_miracle_box"),
|
||||
(1, 2, 3, 1.2, 4, 1.3, segment_zero, "arc_miracle_box"),
|
||||
]
|
||||
|
||||
rows = build_arc_rows_from_pairs(pairs)
|
||||
|
||||
self.assertEqual([row["reward_id"] for row in rows], ["fork_dustbin", "fork_vine"])
|
||||
self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in rows))
|
||||
self.assertEqual(
|
||||
rows[0]["uid"],
|
||||
make_arc_uid(rows[0]["timestamp_raw_hex"], rows[0]["timestamp_group_ordinal"]),
|
||||
)
|
||||
|
||||
def test_matching_structured_data_enriches_without_replacing_heuristic_identity(self):
|
||||
heuristic_payload = fixture_payload("limited-points-gift-1")
|
||||
original = decode_response_records(heuristic_payload)[0]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue