303 lines
12 KiB
Python
303 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import struct
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
|
|
from nte_history_exporter.constants import (
|
|
ARC_BANNER_ID,
|
|
ARC_HISTORY_CURSOR_OFFSET,
|
|
ARC_HISTORY_PAGE_CURSOR_MULTIPLIER,
|
|
ARC_HISTORY_REQUEST_BANNER,
|
|
ARC_HISTORY_REQUEST_LENGTH,
|
|
ARC_RESPONSE_FIRST_RECORD_OFFSET,
|
|
ARC_SYSTEM,
|
|
ARC_TIMESTAMP_TICKS_PER_SECOND,
|
|
DOTNET_UNIX_EPOCH_SECONDS,
|
|
GAME_UID_PART,
|
|
POOL_META,
|
|
)
|
|
from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
|
|
from nte_history_exporter.decoder.run import fmt_packet_time
|
|
from nte_history_exporter.mappings import ARC_META_BY_CASEFOLD
|
|
from nte_history_exporter.decoder.structured_protocol import (
|
|
StructuredProtocolAssembler,
|
|
StructuredRecord,
|
|
parse_structured_blocks,
|
|
parse_structured_records,
|
|
)
|
|
|
|
|
|
def is_arc_history_request(content: bytes) -> bool:
|
|
return len(content) >= ARC_HISTORY_REQUEST_LENGTH and struct.unpack_from("<I", content, 24)[0] == ARC_HISTORY_REQUEST_BANNER
|
|
|
|
|
|
def arc_request_page(content: bytes) -> int:
|
|
return struct.unpack_from("<I", content, ARC_HISTORY_CURSOR_OFFSET)[0] // ARC_HISTORY_PAGE_CURSOR_MULTIPLIER
|
|
|
|
|
|
def decode_arc_key(raw: bytes) -> str | None:
|
|
if raw.endswith(b"\x00"):
|
|
raw = raw[:-1]
|
|
prefix = bytes.fromhex("ccdee4d6be")
|
|
if not raw.startswith(prefix):
|
|
return None
|
|
out = "fork_"
|
|
for byte in raw[len(prefix) :]:
|
|
if 0xC2 <= byte <= 0xF4 and (byte - 0xC2) % 2 == 0:
|
|
out += chr(ord("a") + (byte - 0xC2) // 2)
|
|
elif 0x82 <= byte <= 0xB4 and (byte - 0x82) % 2 == 0:
|
|
out += chr(ord("A") + (byte - 0x82) // 2)
|
|
else:
|
|
out += f"_{byte:02x}"
|
|
return out
|
|
|
|
|
|
def decode_arc_timestamp(raw8: bytes) -> tuple[int, float, str]:
|
|
if len(raw8) != 8:
|
|
raise ValueError("arc timestamps must be exactly 8 bytes")
|
|
ticks = struct.unpack("<Q", raw8)[0]
|
|
unix_seconds = ticks / ARC_TIMESTAMP_TICKS_PER_SECOND - DOTNET_UNIX_EPOCH_SECONDS
|
|
try:
|
|
decoded = datetime.fromtimestamp(unix_seconds, timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
|
except (OverflowError, OSError, ValueError) as exc:
|
|
raise ValueError("arc timestamp is out of range") from exc
|
|
return ticks, unix_seconds, decoded
|
|
|
|
|
|
def _parse_legacy_arc_response(response: bytes) -> list[dict[str, Any]]:
|
|
pos = ARC_RESPONSE_FIRST_RECORD_OFFSET
|
|
records: list[dict[str, Any]] = []
|
|
while pos + 4 <= len(response):
|
|
start = pos
|
|
name_len2 = struct.unpack_from("<I", response, pos)[0]
|
|
pos += 4
|
|
if name_len2 <= 0 or name_len2 > 200 or name_len2 % 2:
|
|
break
|
|
name_len = name_len2 // 2
|
|
if pos + name_len + 4 > len(response):
|
|
break
|
|
name_raw = response[pos : pos + name_len]
|
|
pos += name_len
|
|
|
|
type_len2 = struct.unpack_from("<I", response, pos)[0]
|
|
pos += 4
|
|
if type_len2 <= 0 or type_len2 > 200 or type_len2 % 2:
|
|
break
|
|
type_len = type_len2 // 2
|
|
if pos + type_len + 8 > len(response):
|
|
break
|
|
type_raw = response[pos : pos + type_len]
|
|
pos += type_len
|
|
|
|
timestamp_raw = response[pos : pos + 8]
|
|
pos += 8
|
|
arc_id = decode_arc_key(name_raw) or name_raw.hex()
|
|
canonical_id, meta = _resolve_arc_metadata(arc_id)
|
|
try:
|
|
ticks, unix_seconds, timestamp_decoded = decode_arc_timestamp(timestamp_raw)
|
|
except ValueError:
|
|
return []
|
|
records.append(
|
|
{
|
|
"record_start": start,
|
|
"record_end": pos,
|
|
"record_len": pos - start,
|
|
"reward_key_hex": name_raw.hex(),
|
|
"reward_type": "arc",
|
|
"reward_id": canonical_id,
|
|
"reward_name": meta.get("name", "UNKNOWN"),
|
|
"reward_rank": meta.get("rank", ""),
|
|
"type_key_hex": type_raw.hex(),
|
|
"source_type": "miracle_box",
|
|
"timestamp_raw_hex": timestamp_raw.hex(),
|
|
"timestamp_ticks": ticks,
|
|
"timestamp_unix": unix_seconds,
|
|
"timestamp_decoded": timestamp_decoded,
|
|
"record_hex": response[start:pos].hex(),
|
|
}
|
|
)
|
|
return records
|
|
|
|
|
|
def _resolve_arc_metadata(arc_id: str) -> tuple[str, dict[str, Any]]:
|
|
canonical_id, meta = ARC_META_BY_CASEFOLD.get(arc_id.casefold(), (arc_id, {}))
|
|
return canonical_id, meta
|
|
|
|
|
|
def structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]:
|
|
records = []
|
|
for structured in structured_rows:
|
|
canonical_id, meta = _resolve_arc_metadata(structured.item_id)
|
|
records.append(
|
|
{
|
|
"record_start": structured.record_start,
|
|
"record_end": structured.record_end,
|
|
"record_len": structured.record_end - structured.record_start,
|
|
"reward_key_hex": "",
|
|
"reward_type": "arc",
|
|
"reward_id": canonical_id,
|
|
"reward_name": meta.get("name", "UNKNOWN"),
|
|
"reward_rank": meta.get("rank", ""),
|
|
"type_key_hex": "",
|
|
"source_type": "miracle_box",
|
|
"timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(),
|
|
"timestamp_ticks": structured.ticks,
|
|
"timestamp_unix": structured.timestamp_unix,
|
|
"timestamp_decoded": structured.timestamp_decoded,
|
|
"record_hex": structured.record_hex,
|
|
"decoder_mode": "structured_fallback",
|
|
"structured_pool_id": structured.pool_id,
|
|
"structured_protocol_view": structured.protocol_view,
|
|
"structured_generation_index": structured.generation_index,
|
|
}
|
|
)
|
|
return records
|
|
|
|
|
|
def _enrich_legacy_arc_rows(
|
|
legacy_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord]
|
|
) -> list[dict[str, Any]]:
|
|
if len(legacy_rows) != len(structured_rows):
|
|
return legacy_rows
|
|
for legacy, structured in zip(legacy_rows, structured_rows):
|
|
if legacy.get("reward_id", "").casefold() != structured.item_id.casefold():
|
|
return legacy_rows
|
|
if legacy.get("timestamp_ticks") not in {structured.ticks, structured.ticks * 2}:
|
|
return legacy_rows
|
|
for legacy, structured in zip(legacy_rows, structured_rows):
|
|
legacy["decoder_mode"] = "heuristic_enriched"
|
|
legacy["structured_pool_id"] = structured.pool_id
|
|
legacy["structured_protocol_view"] = structured.protocol_view
|
|
return legacy_rows
|
|
|
|
|
|
def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
|
|
structured_rows = parse_structured_records(response, "fork")
|
|
legacy_rows = _parse_legacy_arc_response(response)
|
|
if legacy_rows:
|
|
return _enrich_legacy_arc_rows(legacy_rows, structured_rows)
|
|
return structured_arc_rows(structured_rows)
|
|
|
|
|
|
def _build_primary_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
|
|
pool = POOL_META["arc_miracle_box"]
|
|
rows: list[dict[str, Any]] = []
|
|
for pair in pairs:
|
|
page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7]
|
|
records = parse_arc_response(response)
|
|
if len(pair) > 9:
|
|
slice_start, slice_count = pair[8:10]
|
|
records = records[slice_start : slice_start + slice_count]
|
|
for row_index, record in enumerate(records, start=1):
|
|
rows.append(
|
|
{
|
|
**record,
|
|
"page": page,
|
|
"offset": offset,
|
|
"row": row_index,
|
|
"pool_group_id": pool["id"],
|
|
"pool_group_name": pool["name"],
|
|
"request_msg": req_i,
|
|
"request_time_utc": fmt_packet_time(req_ts),
|
|
"response_msg": resp_i,
|
|
"response_time_utc": fmt_packet_time(resp_ts),
|
|
"response_len": len(response),
|
|
"record_count": len(records),
|
|
}
|
|
)
|
|
annotate_arc_groups(rows)
|
|
return rows
|
|
|
|
|
|
def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
|
|
primary_rows = _build_primary_arc_rows_from_pairs(pairs)
|
|
if primary_rows and any(row.get("decoder_mode") != "structured_fallback" for row in primary_rows):
|
|
return primary_rows
|
|
|
|
assembler = StructuredProtocolAssembler()
|
|
for source_index, pair in enumerate(pairs):
|
|
assembler.add_blocks(parse_structured_blocks(pair[6], "fork", source_index=source_index))
|
|
assembled = assembler.rows("fork")
|
|
if not assembled:
|
|
return primary_rows
|
|
|
|
pool = POOL_META["arc_miracle_box"]
|
|
rows = []
|
|
for row_index, (record, structured) in enumerate(
|
|
zip(structured_arc_rows(assembled), assembled), start=1
|
|
):
|
|
source_index = structured.source_index or 0
|
|
pair = pairs[source_index]
|
|
page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7]
|
|
rows.append(
|
|
{
|
|
**record,
|
|
"page": page,
|
|
"offset": offset,
|
|
"row": row_index,
|
|
"pool_group_id": pool["id"],
|
|
"pool_group_name": pool["name"],
|
|
"request_msg": req_i,
|
|
"request_time_utc": fmt_packet_time(req_ts),
|
|
"response_msg": resp_i,
|
|
"response_time_utc": fmt_packet_time(resp_ts),
|
|
"response_len": len(response),
|
|
"record_count": len(assembled),
|
|
"structured_assembly": "snapshot_segments",
|
|
"structured_assembly_warning_count": len(assembler.warnings),
|
|
}
|
|
)
|
|
annotate_arc_groups(rows)
|
|
return rows
|
|
|
|
|
|
def make_arc_uid(timestamp_raw: str, ordinal: int) -> str:
|
|
source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal)])
|
|
return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32]
|
|
|
|
|
|
def annotate_arc_groups(rows: list[dict[str, Any]]) -> None:
|
|
# Pages are anchored at page 1, so ordinal 0 of every group is captured and
|
|
# all UIDs are stable -- even a partially captured oldest 10-pull, whose
|
|
# unseen rows can only append after the captured ones. All rows are exported.
|
|
# Arc transactions are always ten records. A transaction can cross response
|
|
# pages whose raw timestamps occasionally differ while displaying the same
|
|
# second. Re-key each consecutive batch to its first timestamp before UIDs
|
|
# are assigned, preserving prefix UIDs and preventing an ordinal restart.
|
|
run_start = 0
|
|
while run_start < len(rows):
|
|
displayed = rows[run_start].get("timestamp_decoded")
|
|
run_end = run_start + 1
|
|
while run_end < len(rows) and rows[run_end].get("timestamp_decoded") == displayed:
|
|
run_end += 1
|
|
for batch_start in range(run_start, run_end, 10):
|
|
batch = rows[batch_start : min(batch_start + 10, run_end)]
|
|
canonical = batch[0].get("timestamp_raw_hex", "")
|
|
for row in batch:
|
|
original = row.get("timestamp_raw_hex", "")
|
|
if original != canonical:
|
|
row["timestamp_reconciled_from_raw_hex"] = original
|
|
row["timestamp_raw_hex"] = canonical
|
|
run_start = run_end
|
|
|
|
groups: dict[str, list[int]] = defaultdict(list)
|
|
for index, row in enumerate(rows):
|
|
groups[row["timestamp_raw_hex"]].append(index)
|
|
|
|
for group_index, (timestamp_raw, indexes) in enumerate(groups.items()):
|
|
for ordinal, index in enumerate(indexes):
|
|
row = rows[index]
|
|
row["timestamp_group_index"] = group_index
|
|
row["timestamp_group_ordinal"] = ordinal
|
|
row["timestamp_group_size_seen"] = len(indexes)
|
|
row["uid"] = make_arc_uid(timestamp_raw, ordinal)
|
|
row["uid_status"] = "stable"
|
|
row["export_record"] = True
|
|
row["skip_reason"] = ""
|
|
|
|
|
|
def select_continuous_arc_run(pairs: list[tuple]) -> tuple[list[tuple], list[dict[str, Any]]]:
|
|
return select_continuous_run_from_page_1(pairs)
|