Initial NTE history exporter
Initial NTE history exporter Add a Windows live-capture exporter for NTE pull history with support for Permanent Monopoly, Limited Monopoly, and Arc Miracle Box histories. Included: - live packet capture via Windows raw sockets - CSV and JSON export generation - normalized reward and banner mapping files - Monopoly v7 Points Gift / Chase Reward classification - Arc Miracle Box decoding with shared pity metadata - timestamp grouping and UID generation - Windows launcher scripts with admin prompting - docs for usage, export format, limitations, and packet notes - tests covering decoder behaviour and mapping consistency Generated exports, captures, briefs, and local data are ignored for privacy.
This commit is contained in:
Golumpa
commit
bb3fe3b19a
35 changed files with 2439 additions and 0 deletions
1
src/nte_history_exporter/live_capture/__init__.py
Normal file
1
src/nte_history_exporter/live_capture/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
140
src/nte_history_exporter/live_capture/runner.py
Normal file
140
src/nte_history_exporter/live_capture/runner.py
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import msvcrt
|
||||
import socket
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from nte_history_exporter.constants import POOL_META
|
||||
from nte_history_exporter.decoder.arc import arc_stability_warnings, select_continuous_arc_run
|
||||
from nte_history_exporter.decoder.boundary import annotate_groups, page_gap_warnings
|
||||
from nte_history_exporter.export.csv_export import write_csv
|
||||
from nte_history_exporter.export.json_export import build_export_json
|
||||
from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket
|
||||
from nte_history_exporter.live_capture.windows_raw import detect_local_ipv4, open_raw_udp_socket, read_packets
|
||||
|
||||
|
||||
EXPORT_PREFIXES = {
|
||||
"permanent": "Permanent",
|
||||
"limited_character": "Limited",
|
||||
"arc_miracle_box": "Arc",
|
||||
}
|
||||
|
||||
|
||||
def copy_to_clipboard(text: str) -> None:
|
||||
try:
|
||||
import tkinter as tk
|
||||
|
||||
root = tk.Tk()
|
||||
root.withdraw()
|
||||
root.clipboard_clear()
|
||||
root.clipboard_append(text)
|
||||
root.update()
|
||||
root.destroy()
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
import subprocess
|
||||
|
||||
subprocess.run(["clip"], input=text, text=True, check=True)
|
||||
|
||||
|
||||
def run_live_capture(
|
||||
*,
|
||||
interface_ip: str | None = None,
|
||||
copy_clipboard: bool = True,
|
||||
) -> dict:
|
||||
local_ip = interface_ip or detect_local_ipv4()
|
||||
session = LiveHistorySession(local_ip)
|
||||
|
||||
sock = open_raw_udp_socket(local_ip)
|
||||
print(f"Device 0 ready~! Listening on {local_ip}")
|
||||
print("Open either Monopoly history board. Capture will stay open until you press any key.")
|
||||
|
||||
try:
|
||||
for packet in read_packets(sock):
|
||||
if msvcrt.kbhit():
|
||||
msvcrt.getch()
|
||||
break
|
||||
if packet is None:
|
||||
continue
|
||||
matched = session.process_packet(
|
||||
UdpPacket(
|
||||
timestamp=time.time(),
|
||||
src_ip=packet.src_ip,
|
||||
dst_ip=packet.dst_ip,
|
||||
src_port=packet.src_port,
|
||||
dst_port=packet.dst_port,
|
||||
payload=packet.payload,
|
||||
)
|
||||
)
|
||||
if matched:
|
||||
kind = session.pairs[-1][7] if session.pairs else "permanent"
|
||||
label = POOL_META.get(kind, POOL_META["permanent"])["name"]
|
||||
print(f"Captured {label} page {session.last_page_seen}")
|
||||
finally:
|
||||
try:
|
||||
sock.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF)
|
||||
except Exception:
|
||||
pass
|
||||
sock.close()
|
||||
|
||||
exports = []
|
||||
for kind in session.kinds_seen():
|
||||
pairs = session.pairs_for_kind(kind)
|
||||
best_run = session.best_run(kind)
|
||||
rows = session.build_rows(kind)
|
||||
if kind == "arc_miracle_box":
|
||||
_arc_run, gap_warnings = select_continuous_arc_run(pairs)
|
||||
warnings = gap_warnings + arc_stability_warnings(rows)
|
||||
else:
|
||||
rows, warnings = annotate_groups(rows, starts_from_page_1=bool(best_run and best_run[0][0] == 1))
|
||||
warnings = page_gap_warnings(pairs, best_run) + warnings
|
||||
pages_seen = [p[0] for p in best_run]
|
||||
csv_path, json_path = export_paths(kind)
|
||||
write_csv(csv_path, rows)
|
||||
export = build_export_json(
|
||||
rows,
|
||||
warnings,
|
||||
source="live_capture",
|
||||
pages_seen=pages_seen,
|
||||
)
|
||||
payload = json.dumps(export, ensure_ascii=False, indent=2)
|
||||
json_path.write_text(payload, encoding="utf-8")
|
||||
exports.append({"kind": kind, "csv_path": csv_path, "json_path": json_path, "export": export, "payload": payload})
|
||||
|
||||
if not exports:
|
||||
print("No Monopoly history pages were captured.")
|
||||
return {"exports": []}
|
||||
|
||||
if copy_clipboard and len(exports) == 1:
|
||||
payload = exports[0]["payload"]
|
||||
copy_to_clipboard(payload)
|
||||
print("Export copied to clipboard.")
|
||||
elif len(exports) > 1:
|
||||
print("Multiple banners captured; clipboard copy skipped.")
|
||||
|
||||
for item in exports:
|
||||
print(f"CSV written: {item['csv_path']}")
|
||||
print(f"Export written: {item['json_path']}")
|
||||
return {"exports": exports}
|
||||
|
||||
|
||||
def export_paths(kind: str) -> tuple[Path, Path]:
|
||||
export_dir = Path("exports")
|
||||
export_dir.mkdir(parents=True, exist_ok=True)
|
||||
prefix = EXPORT_PREFIXES.get(kind, "History")
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
base = export_dir / f"{prefix}_{stamp}"
|
||||
csv_path = base.with_suffix(".csv")
|
||||
json_path = base.with_suffix(".json")
|
||||
counter = 2
|
||||
while csv_path.exists() or json_path.exists():
|
||||
base = export_dir / f"{prefix}_{stamp}_{counter}"
|
||||
csv_path = base.with_suffix(".csv")
|
||||
json_path = base.with_suffix(".json")
|
||||
counter += 1
|
||||
return csv_path, json_path
|
||||
153
src/nte_history_exporter/live_capture/session.py
Normal file
153
src/nte_history_exporter/live_capture/session.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from nte_history_exporter.decoder.arc import (
|
||||
arc_request_page,
|
||||
build_arc_rows_from_pairs,
|
||||
is_arc_history_request,
|
||||
parse_arc_response,
|
||||
select_continuous_arc_run,
|
||||
)
|
||||
from nte_history_exporter.decoder.boundary import longest_monotonic_page_run
|
||||
from nte_history_exporter.decoder.protocol import (
|
||||
history_request_kind,
|
||||
is_history_request,
|
||||
request_page,
|
||||
response_contains_history_marker,
|
||||
)
|
||||
from nte_history_exporter.decoder.run import build_rows_from_pairs
|
||||
|
||||
|
||||
@dataclass
|
||||
class UdpPacket:
|
||||
timestamp: float
|
||||
src_ip: str
|
||||
dst_ip: str
|
||||
src_port: int
|
||||
dst_port: int
|
||||
payload: bytes
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingRequest:
|
||||
page: int
|
||||
offset: int
|
||||
kind: str
|
||||
request_msg: int
|
||||
request_time: float
|
||||
src_ip: str
|
||||
dst_ip: str
|
||||
src_port: int
|
||||
dst_port: int
|
||||
|
||||
|
||||
class LiveHistorySession:
|
||||
def __init__(self, local_ip: str) -> None:
|
||||
self.local_ip = local_ip
|
||||
self.pending: deque[PendingRequest] = deque()
|
||||
self.pairs: list[tuple] = []
|
||||
self.packet_count = 0
|
||||
self.last_match_time: float | None = None
|
||||
self.last_page_seen: int | None = None
|
||||
|
||||
def process_packet(self, packet: UdpPacket) -> bool:
|
||||
self.packet_count += 1
|
||||
|
||||
if packet.src_ip == self.local_ip and is_history_request(packet.payload):
|
||||
offset = int.from_bytes(packet.payload[31:35], "little")
|
||||
req = PendingRequest(
|
||||
page=request_page(packet.payload),
|
||||
offset=offset,
|
||||
kind=history_request_kind(packet.payload),
|
||||
request_msg=self.packet_count,
|
||||
request_time=packet.timestamp,
|
||||
src_ip=packet.src_ip,
|
||||
dst_ip=packet.dst_ip,
|
||||
src_port=packet.src_port,
|
||||
dst_port=packet.dst_port,
|
||||
)
|
||||
self.pending.append(req)
|
||||
self.last_page_seen = req.page
|
||||
return False
|
||||
|
||||
if packet.src_ip == self.local_ip and is_arc_history_request(packet.payload):
|
||||
page = arc_request_page(packet.payload)
|
||||
req = PendingRequest(
|
||||
page=page,
|
||||
offset=page * 2,
|
||||
kind="arc_miracle_box",
|
||||
request_msg=self.packet_count,
|
||||
request_time=packet.timestamp,
|
||||
src_ip=packet.src_ip,
|
||||
dst_ip=packet.dst_ip,
|
||||
src_port=packet.src_port,
|
||||
dst_port=packet.dst_port,
|
||||
)
|
||||
self.pending.append(req)
|
||||
self.last_page_seen = req.page
|
||||
return False
|
||||
|
||||
if packet.dst_ip != self.local_ip or len(packet.payload) < 100:
|
||||
return False
|
||||
|
||||
is_monopoly_response = response_contains_history_marker(packet.payload)
|
||||
is_arc_response = bool(parse_arc_response(packet.payload))
|
||||
if not is_monopoly_response and not is_arc_response:
|
||||
return False
|
||||
|
||||
for req in self.pending:
|
||||
if req.kind == "arc_miracle_box" and not is_arc_response:
|
||||
continue
|
||||
if req.kind != "arc_miracle_box" and not is_monopoly_response:
|
||||
continue
|
||||
if (
|
||||
packet.src_ip == req.dst_ip
|
||||
and packet.dst_ip == req.src_ip
|
||||
and packet.src_port == req.dst_port
|
||||
and packet.dst_port == req.src_port
|
||||
):
|
||||
self.pending.remove(req)
|
||||
self.pairs.append(
|
||||
(
|
||||
req.page,
|
||||
req.offset,
|
||||
req.request_msg,
|
||||
req.request_time,
|
||||
self.packet_count,
|
||||
packet.timestamp,
|
||||
packet.payload,
|
||||
req.kind,
|
||||
)
|
||||
)
|
||||
self.last_match_time = packet.timestamp
|
||||
self.last_page_seen = req.page
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def kinds_seen(self) -> list[str]:
|
||||
seen = []
|
||||
for pair in self.pairs:
|
||||
kind = pair[7] if len(pair) > 7 else "permanent"
|
||||
if kind not in seen:
|
||||
seen.append(kind)
|
||||
return seen
|
||||
|
||||
def pairs_for_kind(self, kind: str) -> list[tuple]:
|
||||
return [pair for pair in self.pairs if (pair[7] if len(pair) > 7 else "permanent") == kind]
|
||||
|
||||
def build_rows(self, kind: str | None = None) -> list[dict[str, Any]]:
|
||||
if kind == "arc_miracle_box":
|
||||
best_run, _warnings = select_continuous_arc_run(self.pairs_for_kind(kind))
|
||||
return build_arc_rows_from_pairs(best_run)
|
||||
return build_rows_from_pairs(self.best_run(kind))
|
||||
|
||||
def best_run(self, kind: str | None = None) -> list[tuple]:
|
||||
pairs = self.pairs_for_kind(kind) if kind else self.pairs
|
||||
if kind == "arc_miracle_box":
|
||||
best_run, _warnings = select_continuous_arc_run(pairs)
|
||||
return best_run
|
||||
return longest_monotonic_page_run(pairs)
|
||||
82
src/nte_history_exporter/live_capture/windows_raw.py
Normal file
82
src/nte_history_exporter/live_capture/windows_raw.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import struct
|
||||
from dataclasses import dataclass
|
||||
|
||||
from nte_history_exporter.live_capture.session import UdpPacket
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedIpUdpPacket:
|
||||
src_ip: str
|
||||
dst_ip: str
|
||||
src_port: int
|
||||
dst_port: int
|
||||
payload: bytes
|
||||
|
||||
|
||||
def detect_local_ipv4() -> str:
|
||||
candidates: list[str] = []
|
||||
try:
|
||||
host = socket.gethostname()
|
||||
for ip in socket.gethostbyname_ex(host)[2]:
|
||||
if not ip.startswith("127."):
|
||||
candidates.append(ip)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe:
|
||||
probe.connect(("1.1.1.1", 80))
|
||||
ip = probe.getsockname()[0]
|
||||
if not ip.startswith("127."):
|
||||
candidates.insert(0, ip)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if not candidates:
|
||||
raise RuntimeError("could not determine a usable local IPv4 address")
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def parse_ipv4_udp_packet(data: bytes) -> ParsedIpUdpPacket | None:
|
||||
if len(data) < 28:
|
||||
return None
|
||||
version_ihl = data[0]
|
||||
if version_ihl >> 4 != 4:
|
||||
return None
|
||||
ihl = (version_ihl & 0x0F) * 4
|
||||
if len(data) < ihl + 8:
|
||||
return None
|
||||
protocol = data[9]
|
||||
if protocol != 17:
|
||||
return None
|
||||
|
||||
src_ip = socket.inet_ntoa(data[12:16])
|
||||
dst_ip = socket.inet_ntoa(data[16:20])
|
||||
src_port, dst_port, udp_len, _checksum = struct.unpack_from("!HHHH", data, ihl)
|
||||
payload = data[ihl + 8 : ihl + udp_len]
|
||||
return ParsedIpUdpPacket(src_ip, dst_ip, src_port, dst_port, payload)
|
||||
|
||||
|
||||
def open_raw_udp_socket(local_ip: str) -> socket.socket:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_IP)
|
||||
sock.bind((local_ip, 0))
|
||||
sock.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
|
||||
sock.ioctl(socket.SIO_RCVALL, socket.RCVALL_ON)
|
||||
sock.settimeout(0.5)
|
||||
return sock
|
||||
|
||||
|
||||
def read_packets(sock: socket.socket):
|
||||
while True:
|
||||
try:
|
||||
data, _addr = sock.recvfrom(65535)
|
||||
except socket.timeout:
|
||||
yield None
|
||||
continue
|
||||
packet = parse_ipv4_udp_packet(data)
|
||||
if packet is None:
|
||||
continue
|
||||
yield packet
|
||||
Loading…
Add table
Add a link
Reference in a new issue