goober/nte-exporter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implemented fallback decoder

Browse source 
Added a new fallback decoder on the chance things do not decode correctly with the primary decoder.
This commit is contained in:
Golumpa 2026-07-15 19:32:40 +01:00
parent 042d47e3d4
commit a5312520e7
9 changed files with 691 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

65
src/nte_history_exporter/decoder/arc.py
View file

@ -22,6 +22,7 @@ from nte_history_exporter.constants import (
from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
from nte_history_exporter.decoder.run import fmt_packet_time
from nte_history_exporter.mappings import ARC_META
from nte_history_exporter.decoder.structured_protocol import StructuredRecord, parse_structured_records
def is_arc_history_request(content: bytes) -> bool:
@ -61,7 +62,7 @@ def decode_arc_timestamp(raw8: bytes) -> tuple[int, float, str]:
return ticks, unix_seconds, decoded
def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
def _parse_legacy_arc_response(response: bytes) -> list[dict[str, Any]]:
pos = ARC_RESPONSE_FIRST_RECORD_OFFSET
records: list[dict[str, Any]] = []
while pos + 4 <= len(response):
@ -116,6 +117,68 @@ def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
return records
def _arc_metadata(arc_id: str) -> dict[str, Any]:
direct = ARC_META.get(arc_id)
if direct is not None:
return direct
folded = arc_id.casefold()
return next((meta for item_id, meta in ARC_META.items() if item_id.casefold() == folded), {})
def _structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]:
records = []
for structured in structured_rows:
meta = _arc_metadata(structured.item_id)
records.append(
{
"record_start": structured.record_start,
"record_end": structured.record_end,
"record_len": structured.record_end - structured.record_start,
"reward_key_hex": "",
"reward_type": "arc",
"reward_id": structured.item_id,
"reward_name": meta.get("name", "UNKNOWN"),
"reward_rank": meta.get("rank", ""),
"type_key_hex": "",
"source_type": "miracle_box",
"timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(),
"timestamp_ticks": structured.ticks,
"timestamp_unix": structured.timestamp_unix,
"timestamp_decoded": structured.timestamp_decoded,
"record_hex": structured.record_hex,
"decoder_mode": "structured_fallback",
"structured_pool_id": structured.pool_id,
"structured_protocol_view": structured.protocol_view,
}
)
return records
def _enrich_legacy_arc_rows(
legacy_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord]
) -> list[dict[str, Any]]:
if len(legacy_rows) != len(structured_rows):
return legacy_rows
for legacy, structured in zip(legacy_rows, structured_rows):
if legacy.get("reward_id", "").casefold() != structured.item_id.casefold():
return legacy_rows
if legacy.get("timestamp_ticks") not in {structured.ticks, structured.ticks * 2}:
return legacy_rows
for legacy, structured in zip(legacy_rows, structured_rows):
legacy["decoder_mode"] = "heuristic_enriched"
legacy["structured_pool_id"] = structured.pool_id
legacy["structured_protocol_view"] = structured.protocol_view
return legacy_rows
def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
structured_rows = parse_structured_records(response, "fork")
legacy_rows = _parse_legacy_arc_response(response)
if legacy_rows:
return _enrich_legacy_arc_rows(legacy_rows, structured_rows)
return _structured_arc_rows(structured_rows)
def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
pool = POOL_META["arc_miracle_box"]
rows: list[dict[str, Any]] = []

120
src/nte_history_exporter/decoder/protocol.py
View file

@ -16,6 +16,10 @@ from nte_history_exporter.constants import (
VALID_DICE_FIELDS,
)
from nte_history_exporter.mappings import REWARDS_BY_ID
from nte_history_exporter.decoder.structured_protocol import (
StructuredRecord,
parse_structured_records,
)
REWARD_ID_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_")
WARP_PIECE_CHASE_PATTERN = bytes.fromhex(
@ -299,7 +303,116 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str,
return rows
def _structured_result(raw: int | None) -> tuple[int | None, int | None, str, int | None]:
if raw is None:
return None, None, "unknown", None
if raw == 0:
return 0, 0, "points_gift", 0
if raw == 0xFFFFFFFF:
return -4, -4, "chase_reward", -4
return raw, raw, "dice", raw
def _structured_rows_compatible(
heuristic_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord]
) -> bool:
if len(heuristic_rows) != len(structured_rows):
return False
for heuristic, structured in zip(heuristic_rows, structured_rows):
heuristic_id = heuristic.get("reward_id") or ""
if heuristic_id and heuristic_id.casefold() != structured.item_id.casefold():
return False
heuristic_ticks = heuristic.get("timestamp_ticks")
if heuristic_ticks not in {structured.ticks, structured.ticks * 4}:
return False
return True
def _enrich_heuristic_rows(
heuristic_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord]
) -> list[dict[str, Any]]:
if not _structured_rows_compatible(heuristic_rows, structured_rows):
return heuristic_rows
for heuristic, structured in zip(heuristic_rows, structured_rows):
if not heuristic.get("reward_id"):
heuristic["reward_id"] = structured.item_id
reward = _reward_metadata(structured.item_id)
heuristic["reward_type"] = reward.get("type") or infer_reward_type(structured.item_id)
heuristic["reward_name"] = reward.get("name", "")
heuristic["reward_rank"] = reward.get("rank")
heuristic["quantity"] = structured.count
if heuristic.get("result_type") == "unknown" or heuristic.get("dice") is None:
dice, dice_raw, result_type, result_source = _structured_result(structured.roll_points_raw)
heuristic["dice"] = dice
heuristic["dice_raw_u32"] = dice_raw
heuristic["result_type"] = result_type
heuristic["result_source_raw"] = result_source
heuristic["roll_result"] = (
"Points Gift"
if result_type == "points_gift"
else ("Chase Reward" if result_type == "chase_reward" else (f"Dice {dice}" if dice else ""))
)
heuristic["decoder_mode"] = "heuristic_enriched"
heuristic["structured_pool_id"] = structured.pool_id
heuristic["secondary_reward_id"] = structured.secondary_item_id
heuristic["secondary_quantity"] = structured.secondary_count
heuristic["structured_protocol_view"] = structured.protocol_view
return heuristic_rows
def _reward_metadata(reward_id: str) -> dict[str, Any]:
direct = REWARDS_BY_ID.get(reward_id)
if direct is not None:
return direct
folded = reward_id.casefold()
return next((meta for item_id, meta in REWARDS_BY_ID.items() if item_id.casefold() == folded), {})
def _structured_monopoly_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]:
rows = []
for row_index, structured in enumerate(structured_rows, start=1):
dice, dice_raw, result_type, result_source = _structured_result(structured.roll_points_raw)
reward = _reward_metadata(structured.item_id)
rows.append(
{
"row": row_index,
"record_start": structured.record_start,
"record_end": structured.record_end,
"record_len": structured.record_end - structured.record_start,
"dice": dice,
"roll_result": (
"Points Gift"
if result_type == "points_gift"
else ("Chase Reward" if result_type == "chase_reward" else (f"Dice {dice}" if dice else ""))
),
"result_type": result_type,
"result_source_raw": result_source,
"dice_raw_u32": dice_raw,
"dice_offset_in_record": None,
"reward_key_hex": "",
"reward_type": reward.get("type") or infer_reward_type(structured.item_id),
"reward_id": structured.item_id,
"reward_name": reward.get("name", ""),
"reward_rank": reward.get("rank"),
"quantity": structured.count,
"timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(),
"timestamp_ticks": structured.ticks,
"timestamp_unix": f"{structured.timestamp_unix:.6f}",
"timestamp_decoded": structured.timestamp_decoded,
"record_hex": structured.record_hex,
"decoder_mode": "structured_fallback",
"structured_pool_id": structured.pool_id,
"secondary_reward_id": structured.secondary_item_id,
"secondary_quantity": structured.secondary_count,
"structured_protocol_view": structured.protocol_view,
}
)
return rows
def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
structured_rows = parse_structured_records(response_content, "monopoly")
heuristic_rows: list[dict[str, Any]] = []
for candidate in iter_history_response_alignments(response_content):
if not any(marker in candidate for marker in MARKERS):
continue
@ -308,5 +421,8 @@ def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
except (OSError, OverflowError, ValueError):
continue
if rows:
return rows
return []
heuristic_rows = rows
break
if heuristic_rows:
return _enrich_heuristic_rows(heuristic_rows, structured_rows)
return _structured_monopoly_rows(structured_rows)

278
src/nte_history_exporter/decoder/structured_protocol.py Normal file
View file

@ -0,0 +1,278 @@
from __future__ import annotations
import struct
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Literal
RecordType = Literal["monopoly", "fork"]
MONOPOLY_MARKER = b"FMonopolyLotteryRecordData"
FORK_MARKER = b"FForkLotteryRecordData"
MAX_ROWS_PER_BLOCK = 100
MAX_STRING_LENGTH = 256
DOTNET_EPOCH_TICKS = 621_355_968_000_000_000
DOTNET_TICKS_PER_SECOND = 10_000_000
MIN_UNIX_SECONDS = 1_500_000_000
MAX_UNIX_SECONDS = 4_102_444_800
class StructuredProtocolError(ValueError):
pass
@dataclass(frozen=True)
class StructuredRecord:
record_type: RecordType
item_id: str
count: int
ticks: int
timestamp_unix: float
timestamp_decoded: str
pool_id: str | None
roll_points_raw: int | None
secondary_item_id: str | None
secondary_count: int | None
record_start: int
record_end: int
record_hex: str
protocol_view: str
def parse_structured_records(payload: bytes, record_type: RecordType) -> list[StructuredRecord]:
"""Parse typed history blocks from raw or bit-shifted protocol payloads.
Invalid candidates are ignored deliberately: callers use this parser only
as enrichment/fallback and retain the established decoder as their primary
path.
"""
marker = MONOPOLY_MARKER if record_type == "monopoly" else FORK_MARKER
for view_name, data in _iter_protocol_views(payload):
if marker not in data:
continue
records: list[StructuredRecord] = []
search_from = 0
while True:
marker_pos = data.find(marker, search_from)
if marker_pos < 0:
break
try:
parsed = _parse_block(data, marker_pos, record_type, marker, view_name)
except (StructuredProtocolError, UnicodeDecodeError):
parsed = []
records.extend(parsed)
search_from = marker_pos + len(marker)
if records:
return records
return []
def _parse_block(
data: bytes,
marker_pos: int,
record_type: RecordType,
marker: bytes,
view_name: str,
) -> list[StructuredRecord]:
pos = marker_pos + len(marker)
if _byte_at(data, pos) == 0:
pos += 1
_reserved = _u32_at(data, pos)
declared_size = _u32_at(data, pos + 4)
row_count = _u32_at(data, pos + 8)
pos += 12
if row_count > MAX_ROWS_PER_BLOCK:
raise StructuredProtocolError(f"row count is too large: {row_count}")
if declared_size > len(data) - pos:
raise StructuredProtocolError("declared block size exceeds payload")
reader = _Reader(data, pos)
records = []
for _row_index in range(row_count):
row_start = reader.pos
if record_type == "monopoly":
record = _parse_monopoly_row(reader, row_start, view_name)
else:
record = _parse_fork_row(reader, row_start, view_name)
records.append(record)
return records
def _parse_monopoly_row(reader: "_Reader", row_start: int, view_name: str) -> StructuredRecord:
roll_points_raw = reader.u32()
item_spec = reader.string()
_reserved = reader.u32()
secondary_count = reader.u32()
secondary_item_id = reader.string()
result_or_pool = reader.string()
pool_pos = reader.pos
possible_pool = reader.try_string()
if possible_pool and possible_pool.startswith("CardPool_"):
pool_id = possible_pool
else:
reader.pos = pool_pos
pool_id = result_or_pool if result_or_pool.startswith("CardPool_") else None
ticks = reader.u64()
return _make_record(
reader,
"monopoly",
item_spec,
ticks,
pool_id,
roll_points_raw,
secondary_item_id or None,
secondary_count,
row_start,
view_name,
)
def _parse_fork_row(reader: "_Reader", row_start: int, view_name: str) -> StructuredRecord:
item_spec = reader.string()
pool_id = reader.string()
ticks = reader.u64()
return _make_record(
reader,
"fork",
item_spec,
ticks,
pool_id or None,
None,
None,
None,
row_start,
view_name,
)
def _make_record(
reader: "_Reader",
record_type: RecordType,
item_spec: str,
ticks: int,
pool_id: str | None,
roll_points_raw: int | None,
secondary_item_id: str | None,
secondary_count: int | None,
row_start: int,
view_name: str,
) -> StructuredRecord:
item_id, count = _parse_item_spec(item_spec)
if not item_id:
raise StructuredProtocolError("structured item ID is empty")
timestamp_unix = (ticks - DOTNET_EPOCH_TICKS) / DOTNET_TICKS_PER_SECOND
if not MIN_UNIX_SECONDS <= timestamp_unix <= MAX_UNIX_SECONDS:
raise StructuredProtocolError("structured timestamp is out of range")
timestamp_decoded = datetime.fromtimestamp(timestamp_unix, timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
return StructuredRecord(
record_type=record_type,
item_id=item_id,
count=count,
ticks=ticks,
timestamp_unix=timestamp_unix,
timestamp_decoded=timestamp_decoded,
pool_id=pool_id,
roll_points_raw=roll_points_raw,
secondary_item_id=secondary_item_id,
secondary_count=secondary_count,
record_start=row_start,
record_end=reader.pos,
record_hex=reader.data[row_start : reader.pos].hex(),
protocol_view=view_name,
)
def _parse_item_spec(value: str) -> tuple[str, int]:
item_id, separator, raw_count = value.rpartition(",")
if separator:
try:
count = int(raw_count)
except ValueError:
count = 0
if item_id and count > 0:
return item_id, count
return value, 1
class _Reader:
def __init__(self, data: bytes, pos: int) -> None:
self.data = data
self.pos = pos
def u32(self) -> int:
value = _u32_at(self.data, self.pos)
self.pos += 4
return value
def u64(self) -> int:
value = _u64_at(self.data, self.pos)
self.pos += 8
return value
def string(self) -> str:
length_pos = self.pos
length = self.u32()
if length == 0 or length > MAX_STRING_LENGTH:
raise StructuredProtocolError(f"invalid string length {length} at {length_pos}")
end = self.pos + length
raw = self.data[self.pos:end]
if len(raw) != length:
raise StructuredProtocolError("string exceeds payload")
self.pos = end
if raw.endswith(b"\0"):
raw = raw[:-1]
return raw.decode("utf-8")
def try_string(self) -> str | None:
start = self.pos
try:
return self.string()
except (StructuredProtocolError, UnicodeDecodeError):
self.pos = start
return None
def _iter_protocol_views(payload: bytes):
yield "raw", payload
for bit_shift in range(1, 8):
shifted = _decode_shifted_bytes(payload, byte_offset=8, bit_shift=bit_shift)
yield f"shift8:{bit_shift}", shifted
def _decode_shifted_bytes(data: bytes, *, byte_offset: int, bit_shift: int) -> bytes:
result = bytearray()
count = max(0, len(data) - byte_offset)
for index in range(count):
bit_pos = (byte_offset + index) * 8 + bit_shift
byte_pos, shift = divmod(bit_pos, 8)
if byte_pos >= len(data):
break
value = data[byte_pos] >> shift
if shift and byte_pos + 1 < len(data):
value |= data[byte_pos + 1] << (8 - shift)
result.append(value & 0xFF)
return bytes(result)
def _byte_at(data: bytes, pos: int) -> int:
try:
return data[pos]
except IndexError as exc:
raise StructuredProtocolError("byte exceeds payload") from exc
def _u32_at(data: bytes, pos: int) -> int:
try:
return struct.unpack_from("<I", data, pos)[0]
except struct.error as exc:
raise StructuredProtocolError("u32 exceeds payload") from exc
def _u64_at(data: bytes, pos: int) -> int:
try:
return struct.unpack_from("<Q", data, pos)[0]
except struct.error as exc:
raise StructuredProtocolError("u64 exceeds payload") from exc

5
src/nte_history_exporter/export/csv_export.py
View file

@ -27,11 +27,16 @@ FIELDNAMES = [
"result_source_raw",
"dice",
"dice_raw_u32",
"decoder_mode",
"structured_protocol_view",
"structured_pool_id",
"reward_type",
"reward_id",
"reward_name",
"reward_rank",
"quantity",
"secondary_reward_id",
"secondary_quantity",
"timestamp_decoded",
"timestamp_raw_hex",
"timestamp_ticks",