from __future__ import annotations import hashlib import struct from collections import defaultdict from datetime import datetime, timezone from typing import Any from nte_history_exporter.constants import ( ARC_BANNER_ID, ARC_HISTORY_CURSOR_OFFSET, ARC_HISTORY_PAGE_CURSOR_MULTIPLIER, ARC_HISTORY_REQUEST_BANNER, ARC_HISTORY_REQUEST_LENGTH, ARC_RESPONSE_FIRST_RECORD_OFFSET, ARC_SYSTEM, ARC_TIMESTAMP_TICKS_PER_SECOND, DOTNET_UNIX_EPOCH_SECONDS, GAME_UID_PART, POOL_META, ) from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 from nte_history_exporter.decoder.run import fmt_packet_time from nte_history_exporter.mappings import ARC_META from nte_history_exporter.decoder.structured_protocol import ( StructuredProtocolAssembler, StructuredRecord, parse_structured_blocks, parse_structured_records, ) def is_arc_history_request(content: bytes) -> bool: return len(content) >= ARC_HISTORY_REQUEST_LENGTH and struct.unpack_from(" int: return struct.unpack_from(" str | None: if raw.endswith(b"\x00"): raw = raw[:-1] prefix = bytes.fromhex("ccdee4d6be") if not raw.startswith(prefix): return None out = "fork_" for byte in raw[len(prefix) :]: if 0xC2 <= byte <= 0xF4 and (byte - 0xC2) % 2 == 0: out += chr(ord("a") + (byte - 0xC2) // 2) elif 0x82 <= byte <= 0xB4 and (byte - 0x82) % 2 == 0: out += chr(ord("A") + (byte - 0x82) // 2) else: out += f"_{byte:02x}" return out def decode_arc_timestamp(raw8: bytes) -> tuple[int, float, str]: if len(raw8) != 8: raise ValueError("arc timestamps must be exactly 8 bytes") ticks = struct.unpack(" list[dict[str, Any]]: pos = ARC_RESPONSE_FIRST_RECORD_OFFSET records: list[dict[str, Any]] = [] while pos + 4 <= len(response): start = pos name_len2 = struct.unpack_from(" 200 or name_len2 % 2: break name_len = name_len2 // 2 if pos + name_len + 4 > len(response): break name_raw = response[pos : pos + name_len] pos += name_len type_len2 = struct.unpack_from(" 200 or type_len2 % 2: break type_len = type_len2 // 2 if pos + type_len + 8 > len(response): break type_raw = response[pos : pos + type_len] pos += type_len timestamp_raw = response[pos : pos + 8] pos += 8 arc_id = decode_arc_key(name_raw) or name_raw.hex() meta = ARC_META.get(arc_id, {}) try: ticks, unix_seconds, timestamp_decoded = decode_arc_timestamp(timestamp_raw) except ValueError: return [] records.append( { "record_start": start, "record_end": pos, "record_len": pos - start, "reward_key_hex": name_raw.hex(), "reward_type": "arc", "reward_id": arc_id, "reward_name": meta.get("name", "UNKNOWN"), "reward_rank": meta.get("rank", ""), "type_key_hex": type_raw.hex(), "source_type": "miracle_box", "timestamp_raw_hex": timestamp_raw.hex(), "timestamp_ticks": ticks, "timestamp_unix": unix_seconds, "timestamp_decoded": timestamp_decoded, "record_hex": response[start:pos].hex(), } ) return records def _arc_metadata(arc_id: str) -> dict[str, Any]: direct = ARC_META.get(arc_id) if direct is not None: return direct folded = arc_id.casefold() return next((meta for item_id, meta in ARC_META.items() if item_id.casefold() == folded), {}) def structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]: records = [] for structured in structured_rows: meta = _arc_metadata(structured.item_id) records.append( { "record_start": structured.record_start, "record_end": structured.record_end, "record_len": structured.record_end - structured.record_start, "reward_key_hex": "", "reward_type": "arc", "reward_id": structured.item_id, "reward_name": meta.get("name", "UNKNOWN"), "reward_rank": meta.get("rank", ""), "type_key_hex": "", "source_type": "miracle_box", "timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(), "timestamp_ticks": structured.ticks, "timestamp_unix": structured.timestamp_unix, "timestamp_decoded": structured.timestamp_decoded, "record_hex": structured.record_hex, "decoder_mode": "structured_fallback", "structured_pool_id": structured.pool_id, "structured_protocol_view": structured.protocol_view, "structured_generation_index": structured.generation_index, } ) return records def _enrich_legacy_arc_rows( legacy_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord] ) -> list[dict[str, Any]]: if len(legacy_rows) != len(structured_rows): return legacy_rows for legacy, structured in zip(legacy_rows, structured_rows): if legacy.get("reward_id", "").casefold() != structured.item_id.casefold(): return legacy_rows if legacy.get("timestamp_ticks") not in {structured.ticks, structured.ticks * 2}: return legacy_rows for legacy, structured in zip(legacy_rows, structured_rows): legacy["decoder_mode"] = "heuristic_enriched" legacy["structured_pool_id"] = structured.pool_id legacy["structured_protocol_view"] = structured.protocol_view return legacy_rows def parse_arc_response(response: bytes) -> list[dict[str, Any]]: structured_rows = parse_structured_records(response, "fork") legacy_rows = _parse_legacy_arc_response(response) if legacy_rows: return _enrich_legacy_arc_rows(legacy_rows, structured_rows) return structured_arc_rows(structured_rows) def _build_primary_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: pool = POOL_META["arc_miracle_box"] rows: list[dict[str, Any]] = [] for pair in pairs: page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7] records = parse_arc_response(response) if len(pair) > 9: slice_start, slice_count = pair[8:10] records = records[slice_start : slice_start + slice_count] for row_index, record in enumerate(records, start=1): rows.append( { **record, "page": page, "offset": offset, "row": row_index, "pool_group_id": pool["id"], "pool_group_name": pool["name"], "request_msg": req_i, "request_time_utc": fmt_packet_time(req_ts), "response_msg": resp_i, "response_time_utc": fmt_packet_time(resp_ts), "response_len": len(response), "record_count": len(records), } ) annotate_arc_groups(rows) return rows def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: primary_rows = _build_primary_arc_rows_from_pairs(pairs) if primary_rows and any(row.get("decoder_mode") != "structured_fallback" for row in primary_rows): return primary_rows assembler = StructuredProtocolAssembler() for source_index, pair in enumerate(pairs): assembler.add_blocks(parse_structured_blocks(pair[6], "fork", source_index=source_index)) assembled = assembler.rows("fork") if not assembled: return primary_rows pool = POOL_META["arc_miracle_box"] rows = [] for row_index, (record, structured) in enumerate( zip(structured_arc_rows(assembled), assembled), start=1 ): source_index = structured.source_index or 0 pair = pairs[source_index] page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7] rows.append( { **record, "page": page, "offset": offset, "row": row_index, "pool_group_id": pool["id"], "pool_group_name": pool["name"], "request_msg": req_i, "request_time_utc": fmt_packet_time(req_ts), "response_msg": resp_i, "response_time_utc": fmt_packet_time(resp_ts), "response_len": len(response), "record_count": len(assembled), "structured_assembly": "snapshot_segments", "structured_assembly_warning_count": len(assembler.warnings), } ) annotate_arc_groups(rows) return rows def make_arc_uid(timestamp_raw: str, ordinal: int) -> str: source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal)]) return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32] def annotate_arc_groups(rows: list[dict[str, Any]]) -> None: # Pages are anchored at page 1, so ordinal 0 of every group is captured and # all UIDs are stable -- even a partially captured oldest 10-pull, whose # unseen rows can only append after the captured ones. All rows are exported. # Arc transactions are always ten records. A transaction can cross response # pages whose raw timestamps occasionally differ while displaying the same # second. Re-key each consecutive batch to its first timestamp before UIDs # are assigned, preserving prefix UIDs and preventing an ordinal restart. run_start = 0 while run_start < len(rows): displayed = rows[run_start].get("timestamp_decoded") run_end = run_start + 1 while run_end < len(rows) and rows[run_end].get("timestamp_decoded") == displayed: run_end += 1 for batch_start in range(run_start, run_end, 10): batch = rows[batch_start : min(batch_start + 10, run_end)] canonical = batch[0].get("timestamp_raw_hex", "") for row in batch: original = row.get("timestamp_raw_hex", "") if original != canonical: row["timestamp_reconciled_from_raw_hex"] = original row["timestamp_raw_hex"] = canonical run_start = run_end groups: dict[str, list[int]] = defaultdict(list) for index, row in enumerate(rows): groups[row["timestamp_raw_hex"]].append(index) for group_index, (timestamp_raw, indexes) in enumerate(groups.items()): for ordinal, index in enumerate(indexes): row = rows[index] row["timestamp_group_index"] = group_index row["timestamp_group_ordinal"] = ordinal row["timestamp_group_size_seen"] = len(indexes) row["uid"] = make_arc_uid(timestamp_raw, ordinal) row["uid_status"] = "stable" row["export_record"] = True row["skip_reason"] = "" def select_continuous_arc_run(pairs: list[tuple]) -> tuple[list[tuple], list[dict[str, Any]]]: return select_continuous_run_from_page_1(pairs)