goober/nte-exporter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial NTE history exporter

Browse source 
Initial NTE history exporter

Add a Windows live-capture exporter for NTE pull history with support for
Permanent Monopoly, Limited Monopoly, and Arc Miracle Box histories.

Included:
- live packet capture via Windows raw sockets
- CSV and JSON export generation
- normalized reward and banner mapping files
- Monopoly v7 Points Gift / Chase Reward classification
- Arc Miracle Box decoding with shared pity metadata
- timestamp grouping and UID generation
- Windows launcher scripts with admin prompting
- docs for usage, export format, limitations, and packet notes
- tests covering decoder behaviour and mapping consistency

Generated exports, captures, briefs, and local data are ignored for privacy.
This commit is contained in:
Golumpa 2026-06-10 23:59:01 +01:00
commit bb3fe3b19a
35 changed files with 2439 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/nte_history_exporter/decoder/__init__.py Normal file
View file

@ -0,0 +1 @@

208
src/nte_history_exporter/decoder/arc.py Normal file
View file

@ -0,0 +1,208 @@
from __future__ import annotations
import hashlib
import struct
from collections import defaultdict
from datetime import datetime, timezone
from typing import Any
from nte_history_exporter.constants import (
ARC_BANNER_ID,
ARC_HISTORY_CURSOR_OFFSET,
ARC_HISTORY_PAGE_CURSOR_MULTIPLIER,
ARC_HISTORY_REQUEST_BANNER,
ARC_HISTORY_REQUEST_LENGTH,
ARC_RESPONSE_FIRST_RECORD_OFFSET,
ARC_SYSTEM,
ARC_TIMESTAMP_TICKS_PER_SECOND,
DOTNET_UNIX_EPOCH_SECONDS,
GAME_UID_PART,
POOL_META,
)
from nte_history_exporter.decoder.boundary import longest_monotonic_page_run
from nte_history_exporter.decoder.run import fmt_packet_time
from nte_history_exporter.mappings import ARC_META
def is_arc_history_request(content: bytes) -> bool:
return len(content) == ARC_HISTORY_REQUEST_LENGTH and struct.unpack_from("<I", content, 24)[0] == ARC_HISTORY_REQUEST_BANNER
def arc_request_page(content: bytes) -> int:
return struct.unpack_from("<I", content, ARC_HISTORY_CURSOR_OFFSET)[0] // ARC_HISTORY_PAGE_CURSOR_MULTIPLIER
def decode_arc_key(raw: bytes) -> str | None:
if raw.endswith(b"\x00"):
raw = raw[:-1]
prefix = bytes.fromhex("ccdee4d6be")
if not raw.startswith(prefix):
return None
out = "fork_"
for byte in raw[len(prefix) :]:
if 0xC2 <= byte <= 0xF4 and (byte - 0xC2) % 2 == 0:
out += chr(ord("a") + (byte - 0xC2) // 2)
elif 0x82 <= byte <= 0xB4 and (byte - 0x82) % 2 == 0:
out += chr(ord("A") + (byte - 0x82) // 2)
else:
out += f"_{byte:02x}"
return out
def decode_arc_timestamp(raw8: bytes) -> tuple[int, float, str]:
ticks = struct.unpack("<Q", raw8)[0]
unix_seconds = ticks / ARC_TIMESTAMP_TICKS_PER_SECOND - DOTNET_UNIX_EPOCH_SECONDS
decoded = datetime.fromtimestamp(unix_seconds, timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
return ticks, unix_seconds, decoded
def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
pos = ARC_RESPONSE_FIRST_RECORD_OFFSET
records: list[dict[str, Any]] = []
while pos + 4 <= len(response):
start = pos
name_len2 = struct.unpack_from("<I", response, pos)[0]
pos += 4
if name_len2 <= 0 or name_len2 > 200 or name_len2 % 2:
break
name_len = name_len2 // 2
if pos + name_len + 4 > len(response):
break
name_raw = response[pos : pos + name_len]
pos += name_len
type_len2 = struct.unpack_from("<I", response, pos)[0]
pos += 4
if type_len2 <= 0 or type_len2 > 200 or type_len2 % 2:
break
type_len = type_len2 // 2
if pos + type_len + 8 > len(response):
break
type_raw = response[pos : pos + type_len]
pos += type_len
timestamp_raw = response[pos : pos + 8]
pos += 8
arc_id = decode_arc_key(name_raw) or name_raw.hex()
meta = ARC_META.get(arc_id, {})
ticks, unix_seconds, timestamp_decoded = decode_arc_timestamp(timestamp_raw)
records.append(
{
"record_start": start,
"record_end": pos,
"record_len": pos - start,
"reward_key_hex": name_raw.hex(),
"reward_type": "arc",
"reward_id": arc_id,
"reward_name": meta.get("name", "UNKNOWN"),
"reward_rank": meta.get("rank", ""),
"type_key_hex": type_raw.hex(),
"source_type": "miracle_box",
"timestamp_raw_hex": timestamp_raw.hex(),
"timestamp_ticks": ticks,
"timestamp_unix": unix_seconds,
"timestamp_decoded": timestamp_decoded,
"record_hex": response[start:pos].hex(),
}
)
return records
def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
pool = POOL_META["arc_miracle_box"]
rows: list[dict[str, Any]] = []
for pair in pairs:
page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7]
records = parse_arc_response(response)
for row_index, record in enumerate(records, start=1):
rows.append(
{
**record,
"page": page,
"offset": offset,
"row": row_index,
"pool_group_id": pool["id"],
"pool_group_name": pool["name"],
"request_msg": req_i,
"request_time_utc": fmt_packet_time(req_ts),
"response_msg": resp_i,
"response_time_utc": fmt_packet_time(resp_ts),
"response_len": len(response),
"record_count": len(records),
}
)
annotate_arc_groups(rows)
return rows
def make_arc_uid(timestamp_raw: str, ordinal: int, arc_key_hex: str) -> str:
source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal), arc_key_hex])
return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32]
def annotate_arc_groups(rows: list[dict[str, Any]]) -> None:
groups: dict[str, list[int]] = defaultdict(list)
for index, row in enumerate(rows):
groups[row["timestamp_raw_hex"]].append(index)
for group_index, (timestamp_raw, indexes) in enumerate(groups.items()):
complete = len(indexes) % 10 == 0
for ordinal, index in enumerate(indexes):
row = rows[index]
row["timestamp_group_index"] = group_index
row["timestamp_group_ordinal"] = ordinal
row["timestamp_group_size_seen"] = len(indexes)
row["uid"] = make_arc_uid(timestamp_raw, ordinal, row["reward_key_hex"])
row["uid_status"] = "stable" if complete else "skipped_incomplete_timestamp_group"
row["export_record"] = complete
row["skip_reason"] = "" if complete else "arc timestamp group is not a complete 10-pull in this capture"
def arc_stability_warnings(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
warnings = []
seen = set()
for row in rows:
if row.get("export_record") is True:
continue
timestamp_raw = row["timestamp_raw_hex"]
if timestamp_raw in seen:
continue
seen.add(timestamp_raw)
group = [r for r in rows if r["timestamp_raw_hex"] == timestamp_raw]
warnings.append(
{
"code": "INCOMPLETE_ARC_10_PULL_DROPPED",
"timestamp_raw": timestamp_raw,
"timestamp_decoded": row["timestamp_decoded"],
"records": len(group),
"reason": "arc timestamp group is not a complete 10-pull in this capture",
}
)
return warnings
def select_continuous_arc_run(pairs: list[tuple]) -> tuple[list[tuple], list[dict[str, Any]]]:
warnings: list[dict[str, Any]] = []
if not pairs:
return [], warnings
pairs_by_page = {pair[0]: pair for pair in pairs}
seen_pages = sorted(pairs_by_page)
if 1 in pairs_by_page:
selected_pages = []
page = 1
while page in pairs_by_page:
selected_pages.append(page)
page += 1
if len(selected_pages) < len(seen_pages):
ignored = [page for page in seen_pages if page not in selected_pages]
warnings.append(
{
"code": "PAGE_GAP_DETECTED",
"ignored_pages": ignored,
"reason": f"Using continuous pages 1-{selected_pages[-1]}; ignored later pages {ignored}.",
}
)
return [pairs_by_page[page] for page in selected_pages], warnings
warnings.append({"code": "DID_NOT_START_AT_PAGE_1", "reason": "Arc history scan did not start at page 1."})
return longest_monotonic_page_run(pairs), warnings

159
src/nte_history_exporter/decoder/boundary.py Normal file
View file

@ -0,0 +1,159 @@
from __future__ import annotations
import hashlib
from typing import Any
from nte_history_exporter.constants import BANNER_ID, GAME_UID_PART, SYSTEM
def make_uid(record: dict[str, Any], ordinal: int) -> str:
source = "|".join(
[
GAME_UID_PART,
SYSTEM,
str(record.get("pool_group_id", BANNER_ID)),
str(record.get("timestamp_raw_hex", "")),
str(ordinal),
str(record.get("dice", "")),
str(record.get("reward_key_hex", "")),
str(record.get("quantity", "")),
]
)
return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32]
def longest_monotonic_page_run(pairs: list[tuple]) -> list[tuple]:
runs: list[list[tuple]] = []
current: list[tuple] = []
prev_page = None
for pair in pairs:
page = pair[0]
if prev_page is None or page == prev_page + 1:
current.append(pair)
else:
if current:
runs.append(current)
current = [pair]
prev_page = page
if current:
runs.append(current)
return max(runs, key=len) if runs else []
def page_gap_warnings(pairs: list[tuple], best_run: list[tuple]) -> list[dict[str, Any]]:
warnings: list[dict[str, Any]] = []
if len(pairs) < 2:
return warnings
best_run_ids = {id(pair) for pair in best_run}
ignored_pages = [pair[0] for pair in pairs if id(pair) not in best_run_ids]
previous_page = pairs[0][0]
for pair in pairs[1:]:
page = pair[0]
if page != previous_page + 1:
warning = {
"code": "PAGE_GAP_DETECTED",
"previous_page": previous_page,
"next_page": page,
"ignored_pages": ignored_pages,
"reason": (
f"Page gap detected: saw page {previous_page} then page {page}. "
"Pages outside the longest continuous run were ignored for stable dedupe. "
"Re-scan or scroll more slowly."
),
}
warnings.append(warning)
previous_page = page
return warnings
def is_dice_record(row: dict[str, Any]) -> bool:
result_type = row.get("result_type")
if result_type:
return result_type == "dice"
dice = row.get("dice")
if dice in ("", None):
return False
try:
return int(dice) > 0
except (TypeError, ValueError):
return False
def annotate_groups(
rows: list[dict[str, Any]],
*,
starts_from_page_1: bool = True,
stable_only: bool = True,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
if not rows:
return rows, []
last_page = max(int(r["page"]) for r in rows if str(r.get("page", "")).isdigit())
last_page_records = [r for r in rows if r.get("page") == last_page]
final_page_is_partial = len(last_page_records) < 5
groups: list[list[dict[str, Any]]] = []
current: list[dict[str, Any]] = []
prev_ts = None
for row in rows:
ts = row.get("timestamp_raw_hex", "")
if prev_ts is None or ts == prev_ts:
current.append(row)
else:
groups.append(current)
current = [row]
prev_ts = ts
if current:
groups.append(current)
warnings: list[dict[str, Any]] = []
for group_index, group in enumerate(groups):
at_newest_boundary = group_index == 0
at_oldest_boundary = group_index == len(groups) - 1
dice_records_in_group = [row for row in group if is_dice_record(row)]
dice_record_count = len(dice_records_in_group)
group_status = "stable"
skip_reason = ""
if at_newest_boundary and not starts_from_page_1:
group_status = "dropped_boundary_group"
skip_reason = "newest timestamp group may be partial because scan did not start from page 1"
elif at_oldest_boundary and not final_page_is_partial:
group_status = "dropped_boundary_group"
skip_reason = "oldest timestamp group may continue onto the next uncaptured page"
if group_status != "stable":
warnings.append(
{
"code": "PARTIAL_TIMESTAMP_GROUP_DROPPED",
"timestamp_raw": group[0].get("timestamp_raw_hex", ""),
"timestamp_decoded": group[0].get("timestamp_decoded", ""),
"records": len(group),
"dice_records": dice_record_count,
"reason": skip_reason,
}
)
for ordinal, row in enumerate(group):
row["timestamp_group_index"] = group_index
row["timestamp_group_ordinal"] = ordinal
row["timestamp_group_size_seen"] = dice_record_count
row["timestamp_group_record_size_seen"] = len(group)
row["timestamp_group_boundary"] = ",".join(
name
for name, yes in [("newest", at_newest_boundary), ("oldest", at_oldest_boundary)]
if yes
)
if group_status == "stable":
row["uid_status"] = "stable"
row["uid"] = make_uid(row, ordinal)
row["export_record"] = True
row["skip_reason"] = ""
else:
row["uid_status"] = group_status
row["uid"] = "" if stable_only else make_uid(row, ordinal)
row["export_record"] = not stable_only
row["skip_reason"] = skip_reason
return rows, warnings

179
src/nte_history_exporter/decoder/protocol.py Normal file
View file

@ -0,0 +1,179 @@
from __future__ import annotations
import struct
from datetime import datetime, timezone
from typing import Any
from nte_history_exporter.constants import (
DOTNET_UNIX_EPOCH_SECONDS,
HISTORY_REQUEST_BANNER,
HISTORY_REQUEST_LENGTH,
HISTORY_PAGE_CURSOR_MULTIPLIER,
LIMITED_CHARACTER_SELECTOR,
MARKERS,
PERMANENT_SELECTOR,
TIMESTAMP_TICKS_PER_SECOND,
VALID_DICE_FIELDS,
)
from nte_history_exporter.mappings import KNOWN_REWARDS
def decode_history_timestamp(raw8: bytes) -> tuple[int, float, str]:
if len(raw8) != 8:
raise ValueError("history timestamps must be exactly 8 bytes")
ticks = struct.unpack("<Q", raw8)[0]
unix_seconds = ticks / TIMESTAMP_TICKS_PER_SECOND - DOTNET_UNIX_EPOCH_SECONDS
decoded = datetime.fromtimestamp(unix_seconds, timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
return ticks, unix_seconds, decoded
def history_request_kind(content: bytes) -> str:
if len(content) != HISTORY_REQUEST_LENGTH or struct.unpack_from("<I", content, 35)[0] != HISTORY_REQUEST_BANNER:
return ""
selector = struct.unpack_from("<I", content, 40)[0]
if selector == PERMANENT_SELECTOR:
return "permanent"
if selector == LIMITED_CHARACTER_SELECTOR:
return "limited_character"
return ""
def is_history_request(content: bytes) -> bool:
return bool(history_request_kind(content))
def request_page(content: bytes) -> int:
return struct.unpack_from("<I", content, 31)[0] // HISTORY_PAGE_CURSOR_MULTIPLIER
def response_contains_history_marker(content: bytes) -> bool:
return any(marker in content for marker in MARKERS)
def extract_key(chunk_without_marker: bytes) -> str:
fashion_prefix = bytes.fromhex("1885cda1a5bdb97d")
fashion_pos = chunk_without_marker.rfind(fashion_prefix)
if fashion_pos != -1:
return chunk_without_marker[fashion_pos:].hex()
char_prefix = bytes.fromhex("c4c0")
char_pos = chunk_without_marker.find(char_prefix)
if char_pos != -1 and char_pos + 5 <= len(chunk_without_marker):
return chunk_without_marker[char_pos : char_pos + 5].hex()
best = None
for prefix in [bytes.fromhex("98bdc9ad"), bytes.fromhex("10a58d95")]:
pos = chunk_without_marker.rfind(prefix)
if pos != -1 and (best is None or pos > best):
best = pos
return "" if best is None else chunk_without_marker[best:].hex()
def extract_dice(chunk_without_marker: bytes) -> tuple[int | None, int | None, int | None]:
for off in range(0, min(16, max(0, len(chunk_without_marker) - 3))):
val = struct.unpack_from("<I", chunk_without_marker, off)[0]
if val in VALID_DICE_FIELDS:
return (0 if val == 0 else val // 4), val, off
return None, None, None
def classify_result_type(
chunk_without_marker: bytes,
dice: int | None,
dice_offset: int | None,
) -> tuple[str, int | None]:
if dice is None or dice_offset is None:
return "unknown", None
if dice == 0:
return "points_gift", 0
source_off = dice_offset + 4
if source_off + 4 <= len(chunk_without_marker):
source_val = struct.unpack_from("<i", chunk_without_marker, source_off)[0]
if source_val == 0:
return "points_gift", source_val
if source_val == -4:
return "chase_reward", source_val
return "dice", source_val
return "dice", None
def guess_quantity(chunk_hex: str, key_hex: str, result_type: str | None = None) -> int | None:
if key_hex == "10a58d957dd1a58dad95d17dc1c400":
if result_type == "chase_reward":
return 30
return 4
if key_hex == "10a58d9539bdc9b585b101":
return 1
if key_hex == "10a58d957dd1a58dad95d17dc1c800":
if "c8b0d4c0" in chunk_hex:
return 50
if "c8b0ccc0" in chunk_hex:
return 30
return None
if key_hex:
return 1
return None
def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
marker = b""
marker_offsets: list[int] = []
for candidate_marker in MARKERS:
offsets = [i for i in range(len(response_content)) if response_content.startswith(candidate_marker, i)]
if offsets:
marker = candidate_marker
marker_offsets = offsets
break
if not marker_offsets:
return []
rows: list[dict[str, Any]] = []
prev = 0x50
for row_index, marker_offset in enumerate(marker_offsets, start=1):
chunk = response_content[prev:marker_offset]
full_record = response_content[prev : marker_offset + len(marker) + 8]
dice, dice_raw, dice_offset = extract_dice(chunk)
result_type, result_source_raw = classify_result_type(chunk, dice, dice_offset)
if result_type == "points_gift":
dice = 0
dice_raw = 0
elif result_type == "chase_reward":
dice = -4
dice_raw = -4
key_hex = extract_key(chunk)
reward = KNOWN_REWARDS.get(key_hex, {})
timestamp_raw = response_content[marker_offset + len(marker) : marker_offset + len(marker) + 8]
timestamp_ticks, timestamp_unix, timestamp_decoded = decode_history_timestamp(timestamp_raw)
chunk_hex = chunk.hex()
rows.append(
{
"row": row_index,
"record_start": prev,
"record_end": marker_offset + len(marker) + 8,
"record_len": len(full_record),
"dice": dice,
"roll_result": (
"Points Gift"
if result_type == "points_gift"
else ("Chase Reward" if result_type == "chase_reward" else (f"Dice {dice}" if dice else ""))
),
"result_type": result_type,
"result_source_raw": result_source_raw,
"dice_raw_u32": dice_raw,
"dice_offset_in_record": dice_offset,
"reward_key_hex": key_hex,
"reward_type": reward.get("type", ""),
"reward_id": reward.get("id", ""),
"reward_name": reward.get("name", ""),
"reward_rank": reward.get("rank"),
"quantity": guess_quantity(chunk_hex, key_hex, result_type),
"timestamp_raw_hex": timestamp_raw.hex(),
"timestamp_ticks": timestamp_ticks,
"timestamp_unix": f"{timestamp_unix:.6f}",
"timestamp_decoded": timestamp_decoded,
"record_hex": full_record.hex(),
}
)
prev = marker_offset + len(marker) + 8
return rows

57
src/nte_history_exporter/decoder/run.py Normal file
View file

@ -0,0 +1,57 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from nte_history_exporter.constants import POOL_META
from nte_history_exporter.decoder.protocol import decode_response_records
def fmt_packet_time(ts: float | None) -> str:
if ts is None:
return ""
return datetime.fromtimestamp(ts, timezone.utc).strftime("%H:%M:%S.%f")[:-3]
def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
rows_out: list[dict[str, Any]] = []
for pair in pairs:
page, offset, req_i, req_ts, resp_i, resp_ts, response_content = pair[:7]
kind = pair[7] if len(pair) > 7 else "permanent"
pool = POOL_META.get(kind, POOL_META["permanent"])
records = decode_response_records(response_content)
if not records:
rows_out.append(
{
"page": page,
"offset": offset,
"row": "",
"pool_group_id": pool["id"],
"pool_group_name": pool["name"],
"request_msg": req_i,
"request_time_utc": fmt_packet_time(req_ts),
"response_msg": resp_i,
"response_time_utc": fmt_packet_time(resp_ts),
"response_len": len(response_content),
"record_count": 0,
"record_hex": response_content.hex(),
}
)
continue
for record in records:
rows_out.append(
{
"page": page,
"offset": offset,
"pool_group_id": pool["id"],
"pool_group_name": pool["name"],
"request_msg": req_i,
"request_time_utc": fmt_packet_time(req_ts),
"response_msg": resp_i,
"response_time_utc": fmt_packet_time(resp_ts),
"response_len": len(response_content),
"record_count": len(records),
**record,
}
)
return rows_out