From df1add6cf1657f2bb7da1023bb3582f9f23b59f3 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Wed, 17 Jun 2026 11:20:31 +0100 Subject: [PATCH 1/3] 0.1.3 - Add user UID, capture source & TCP support Introduce automatic user UID extraction and optional --user-uid override, include user_uid and capture_source in exported JSON, and prepend UID to export filenames (falls back to unknown). Propagate detected protocol (tcp/udp) through live/libpcap parsing and accept TCP packets for UID detection; adjust libpcap filter to capture TCP as well. Update CLI flags (--copy-clipboard semantics, --user-uid), prompt user when UID is missing, and wire UID through mitmproxy and live capture flows. Bump package/exporter version to 0.1.3 and add tests for UID extraction and filename behavior. --- README.md | 20 +++-- docs/export-format.md | 7 +- pyproject.toml | 2 +- src/nte_history_exporter/__init__.py | 2 +- .../adapters/mitmproxy_flows.py | 9 ++ src/nte_history_exporter/cli.py | 14 ++- src/nte_history_exporter/console.py | 17 +++- src/nte_history_exporter/constants.py | 2 +- src/nte_history_exporter/decoder/user_uid.py | 46 ++++++++++ .../export/json_export.py | 32 ++++--- .../live_capture/libpcap.py | 6 +- .../live_capture/runner.py | 39 +++++++-- .../live_capture/session.py | 12 ++- .../live_capture/windows_raw.py | 37 ++++++-- tests/test_boundary_export.py | 87 +++++++++++++++++++ 15 files changed, 283 insertions(+), 49 deletions(-) create mode 100644 src/nte_history_exporter/decoder/user_uid.py diff --git a/README.md b/README.md index f2d9f21..063dd14 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Prototype CLI exporter for **Neverness to Everness** pull history — decodes yo The exporter decodes Permanent Board, Limited Character Board, and Arc Miracle Box history pages from captured UDP data, applies conservative timestamp-boundary handling, and writes sanitized JSON suitable for tracker import. > [!NOTE] -> The import JSON contains decoded history rows only. It does **not** export tokens, account IDs, role IDs, device IDs, server IPs, raw packets, cookies, session data, or any other capture metadata. +> The import JSON contains decoded history rows and the shareable NTE user UID when it can be detected. It does **not** export tokens, account IDs, role IDs, device IDs, server IPs, raw packets, cookies, session data, or other capture metadata. ## Requirements @@ -57,15 +57,17 @@ Use `--capture-backend libpcap` to require Npcap/libpcap without fallback, or `- Or simply double-click **`run-exporter.cmd`** — it asks for confirmation before requesting Administrator privileges, and does nothing until you agree. > [!IMPORTANT] -> Launch the tool **before pressing Start on the game's main menu** so the game's UDP connection can be captured. If you are already in game, log out to the main menu and enter again. +> For automatic user UID detection, launch the tool **before pressing Start on the game's main menu**. If you are already in game, history capture can still work; the tool will ask for your UID before saving if it cannot detect it automatically. Once running, open any supported history board in game. The tool keeps listening until you press any key. Exports are written under `exports\` as: -- `Permanent_.json` -- `Limited_.json` -- `Arc_.json` +- `_Permanent_.json` +- `_Limited_.json` +- `_Arc_.json` -If only one banner is captured, the export JSON is copied to your clipboard. If multiple banners are captured in the same run, clipboard copy is skipped so one banner does not overwrite another. +If the user UID is not detected automatically, the console asks for it before saving. Leaving it blank saves as `unknown__.json`, but may prevent import on some trackers. + +Exports are not copied to the clipboard by default. Add `--copy-clipboard` to copy a single captured banner's JSON after saving. If multiple banners are captured in the same run, clipboard copy is skipped so one banner does not overwrite another. If a page response is missed, the exporter reports the missing page number while capture is still running. Leave the exporter open, close and reopen that history board, then scroll down again. Scrolling backward within the existing view does not request the cached pages again. The replacement capture is accepted and the tool confirms when the gap has been recovered. If reopening the board still produces no page messages, return to the main menu and re-enter the game to start a fresh connection. @@ -96,6 +98,8 @@ Decodes a `mitmproxy .flows` capture instead of listening live — used for rese | --------- | --------------------------------------------------- | | `--live` | Capture live UDP traffic instead of reading a file. | | `--debug` | Also write the full research CSV next to each JSON. | +| `--user-uid ` | Override the auto-detected NTE user UID in the JSON export. | +| `--copy-clipboard` | Copy a single live export JSON to clipboard after saving. | Advanced live-capture selection: @@ -107,13 +111,15 @@ Advanced live-capture selection: The `--debug` CSV holds any extra information that might be needed for fixing bugs. It contains no dangerous personal account data — only the raw bytes of the captured history page. +The exporter automatically includes the shareable NTE user UID when it appears in the capture. If a short capture does not include it, the console asks before saving; you can also pass it explicitly with `--user-uid`. + > [!TIP] > For reliable deduplication, start from page 1 and scroll through the pages. If you only want pages 1–5, scroll through to page 6 as well just to be on the safe side. ## Privacy > [!CAUTION] -> Do not commit packet captures, generated exports, research briefs, or personal account data. The repository keeps `exports/` as an empty output folder but ignores everything generated inside it. +> Sanitized exports are intended for tracker import and should not contain anything especially harmful, but they can identify the game account via user UID and pull history. Share exports only with verified sources, such as known trackers. Do not commit packet captures, generated exports, research briefs, or personal account data. The repository keeps `exports/` as an empty output folder but ignores everything generated inside it. ## Boundary Policy diff --git a/docs/export-format.md b/docs/export-format.md index c1181f8..a37f10c 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -8,9 +8,10 @@ The sanitized JSON export uses: "format_version": 1, "game": "Neverness to Everness", "source": "packet_capture", + "capture_source": "npcap", "exporter": { "name": "nte-history-exporter", - "version": "0.1.1" + "version": "0.1.3" }, "banner": { "id": "Lottery_Permanent", @@ -26,6 +27,7 @@ The sanitized JSON export uses: "skipped_records": 0, "warnings": [] }, + "user_uid": "optional-user-uid", "records": [] } ``` @@ -85,3 +87,6 @@ Example Arc record: ``` Normal JSON exports do not include raw packets or capture-only metadata. +`user_uid` is included when detected automatically or supplied with `--user-uid`. +`capture_source` records the capture backend/source used for the export, such as +`npcap`, `libpcap`, `windows_packet`, or `mitmproxy_flows`. diff --git a/pyproject.toml b/pyproject.toml index 7a412f1..d9f6034 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nte-history-exporter" -version = "0.1.2" +version = "0.1.3" description = "Cross-platform Neverness to Everness pull-history exporter." requires-python = ">=3.10" dependencies = [] diff --git a/src/nte_history_exporter/__init__.py b/src/nte_history_exporter/__init__.py index b3f4756..ae73625 100644 --- a/src/nte_history_exporter/__init__.py +++ b/src/nte_history_exporter/__init__.py @@ -1 +1 @@ -__version__ = "0.1.2" +__version__ = "0.1.3" diff --git a/src/nte_history_exporter/adapters/mitmproxy_flows.py b/src/nte_history_exporter/adapters/mitmproxy_flows.py index 03bc23c..19c35f7 100644 --- a/src/nte_history_exporter/adapters/mitmproxy_flows.py +++ b/src/nte_history_exporter/adapters/mitmproxy_flows.py @@ -1,11 +1,13 @@ from __future__ import annotations +from collections import Counter from pathlib import Path from typing import Any from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1 from nte_history_exporter.decoder.arc import build_arc_rows_from_pairs, select_continuous_arc_run from nte_history_exporter.decoder.run import build_rows_from_pairs +from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket @@ -77,6 +79,12 @@ def find_udp_flow(flows: list[Any], preferred_index: int | None = None) -> tuple def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> dict[str, Any]: flows = read_flows(path) + user_uid_candidates: Counter[str] = Counter() + for flow in flows: + for msg in flow.get(b"messages", []): + user_uid_candidates.update(extract_user_uid_candidates(msg[1])) + user_uid = user_uid_candidates.most_common(1)[0][0] if user_uid_candidates else None + resolved_flow_index, flow = find_udp_flow(flows, flow_index) messages = flow[b"messages"] @@ -110,4 +118,5 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d "best_arc_run": best_arc_run, "arc_rows": arc_rows, "arc_warnings": arc_warnings, + "user_uid": session.user_uid or user_uid, } diff --git a/src/nte_history_exporter/cli.py b/src/nte_history_exporter/cli.py index 60292ca..426b564 100644 --- a/src/nte_history_exporter/cli.py +++ b/src/nte_history_exporter/cli.py @@ -26,8 +26,9 @@ def build_parser() -> argparse.ArgumentParser: default="auto", help="live capture backend; auto prefers Npcap/libpcap and falls back to raw sockets on Windows", ) - parser.add_argument("--no-clipboard", action="store_true", help="do not copy live exports to clipboard") + parser.add_argument("--copy-clipboard", action="store_true", help="copy a single live export to clipboard") parser.add_argument("--debug", action="store_true", help="also write research CSVs next to the JSON exports") + parser.add_argument("--user-uid", default=None, help="override the auto-detected NTE user UID in the JSON export") return parser @@ -39,8 +40,9 @@ def main(argv: list[str] | None = None) -> int: run_live_capture( interface_ip=args.interface_ip, capture_backend=args.capture_backend, - copy_clipboard=not args.no_clipboard, + copy_clipboard=args.copy_clipboard, write_debug_csv=args.debug, + user_uid=args.user_uid, ) return 0 except (LibpcapUnavailable, PermissionError) as exc: @@ -62,13 +64,19 @@ def main(argv: list[str] | None = None) -> int: best_run = decoded["best_run"] pair_count = len(decoded["pairs"]) - out_path, json_path = export_paths(kind) + resolved_user_uid = args.user_uid or decoded.get("user_uid") + if not resolved_user_uid: + resolved_user_uid = console.prompt_user_uid() + + out_path, json_path = export_paths(kind, resolved_user_uid) if args.debug: write_csv(out_path, rows) export = build_export_json( rows, warnings, source="packet_capture", + capture_source="mitmproxy_flows", + user_uid=resolved_user_uid, flow_index=decoded["flow_index"], candidate_request_response_pairs=pair_count, pages_seen=[p[0] for p in best_run], diff --git a/src/nte_history_exporter/console.py b/src/nte_history_exporter/console.py index c615aad..c92fb41 100644 --- a/src/nte_history_exporter/console.py +++ b/src/nte_history_exporter/console.py @@ -68,10 +68,10 @@ def print_live_instructions(local_ip: str, backend: str = "windows_raw", detail: print(style(f" Capture backend: {backend}{backend_detail}", DIM)) print() print(style(" How to export your pull history", BOLD)) - print(" 1. This tool must be running BEFORE you press Start on") - print(" the game's main menu, or the game connection cannot") - print(" be captured. Already in game? Log out to the main") - print(" menu and enter again.") + print(" 1. For automatic user UID detection, start this tool") + print(" before pressing Start on the game's main menu.") + print(" Already in game? You can still capture history;") + print(" the tool will ask for your UID if it cannot detect it.") print(" 2. Open a supported history screen:") print(style(" Monopoly > Standard Board history", CYAN)) print(style(" Monopoly > Limited Character Board history", CYAN)) @@ -151,3 +151,12 @@ def print_success(text: str) -> None: def print_problem(text: str) -> None: print(style(f" {text}", YELLOW, BOLD)) + + +def prompt_user_uid() -> str | None: + print() + print_problem("User UID was not detected in this capture.") + print_note("Enter your NTE user UID so the export can be named and linked correctly.") + print_note("Leaving this blank may prevent import on some trackers.") + value = input(" User UID: ").strip() + return value or None diff --git a/src/nte_history_exporter/constants.py b/src/nte_history_exporter/constants.py index 11a9500..fc1248c 100644 --- a/src/nte_history_exporter/constants.py +++ b/src/nte_history_exporter/constants.py @@ -11,7 +11,7 @@ LIMITED_CHARACTER_BANNER_NAME = "Limited Character Board" ARC_BANNER_ID = "Arc_MiracleBox" ARC_BANNER_NAME = "Arc Miracle Box" EXPORTER_NAME = "nte-history-exporter" -EXPORTER_VERSION = "0.1.2" +EXPORTER_VERSION = "0.1.3" HISTORY_REQUEST_BANNER = 4220 HISTORY_REQUEST_LENGTH = 45 diff --git a/src/nte_history_exporter/decoder/user_uid.py b/src/nte_history_exporter/decoder/user_uid.py new file mode 100644 index 0000000..7746b27 --- /dev/null +++ b/src/nte_history_exporter/decoder/user_uid.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import struct +from collections import Counter + + +MIN_USER_UID = 100_000_000_000 +MAX_USER_UID = 999_999_999_999 +USER_UID_RECORD_OFFSETS = ( + (b"TagOthers", (16,)), + (b"PrivateSpawnInfoRecord", (40,)), + (b"SimpleQuestRecord", (20, 52)), + (b"FurnitureLayoutDataRec", (56, 64)), + (b"StoreBrandItemSalesVolumeRecord", (16,)), + (b"StorePropertyRecord", (20,)), +) + + +def _plausible_user_uid(value: int) -> bool: + return MIN_USER_UID <= value <= MAX_USER_UID + + +def extract_user_uid_candidates(payload: bytes) -> list[str]: + results = [] + for marker, distances in USER_UID_RECORD_OFFSETS: + search_from = 0 + while True: + marker_pos = payload.find(marker, search_from) + if marker_pos == -1: + break + for distance in distances: + offset = marker_pos - distance + if offset < 0: + continue + value = struct.unpack_from(" str | None: + candidates = extract_user_uid_candidates(payload) + if not candidates: + return None + return Counter(candidates).most_common(1)[0][0] diff --git a/src/nte_history_exporter/export/json_export.py b/src/nte_history_exporter/export/json_export.py index 271b08c..2735136 100644 --- a/src/nte_history_exporter/export/json_export.py +++ b/src/nte_history_exporter/export/json_export.py @@ -17,6 +17,8 @@ def build_export_json( warnings: list[dict[str, Any]], *, source: str = "packet_capture", + capture_source: str | None = None, + user_uid: str | None = None, flow_index: int | None = None, candidate_request_response_pairs: int | None = None, pages_seen: list[int] | None = None, @@ -40,21 +42,31 @@ def build_export_json( if pages_seen is not None: scan["pages_seen"] = pages_seen - return { + normalized_user_uid = user_uid.strip() if user_uid else "" + export: dict[str, Any] = { "format": "nte-history-export", "format_version": 1, "game": GAME_NAME, "source": source, - "exporter": {"name": EXPORTER_NAME, "version": __version__}, - "banner": { - "id": pool["id"], - "name": pool["name"], - "system": pool["system"], - "shared_pity": pool["shared_pity"], - }, - "scan": scan, - "records": [_record_for_export(r) for r in exported], } + if capture_source: + export["capture_source"] = capture_source + export["exporter"] = {"name": EXPORTER_NAME, "version": __version__} + export.update( + { + "banner": { + "id": pool["id"], + "name": pool["name"], + "system": pool["system"], + "shared_pity": pool["shared_pity"], + }, + "scan": scan, + } + ) + if normalized_user_uid: + export["user_uid"] = normalized_user_uid + export["records"] = [_record_for_export(r) for r in exported] + return export def _record_for_export(row: dict[str, Any]) -> dict[str, Any]: diff --git a/src/nte_history_exporter/live_capture/libpcap.py b/src/nte_history_exporter/live_capture/libpcap.py index 3cb27bd..ba78f06 100644 --- a/src/nte_history_exporter/live_capture/libpcap.py +++ b/src/nte_history_exporter/live_capture/libpcap.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Iterator -from nte_history_exporter.live_capture.windows_raw import ParsedIpUdpPacket, parse_ipv4_udp_packet +from nte_history_exporter.live_capture.windows_raw import ParsedIpUdpPacket, parse_ipv4_packet PCAP_ERRBUF_SIZE = 256 SNAP_LENGTH = 65535 @@ -351,7 +351,7 @@ class LibpcapCapture: raise LibpcapUnavailable(_pcap_error(self.lib, self.handle)) program = _BpfProgram() - filter_expression = f"udp and host {local_ip}".encode("ascii") + filter_expression = f"host {local_ip} and (udp or tcp)".encode("ascii") if self.lib.pcap_compile(self.handle, ctypes.byref(program), filter_expression, 1, 0xFFFFFFFF) != 0: raise LibpcapUnavailable(_pcap_error(self.lib, self.handle)) try: @@ -382,7 +382,7 @@ class LibpcapCapture: ipv4_packet = _extract_ipv4_frame(frame, self.datalink) if ipv4_packet is None: continue - packet = parse_ipv4_udp_packet(ipv4_packet) + packet = parse_ipv4_packet(ipv4_packet) if packet is not None: yield packet diff --git a/src/nte_history_exporter/live_capture/runner.py b/src/nte_history_exporter/live_capture/runner.py index 5daa138..7763867 100644 --- a/src/nte_history_exporter/live_capture/runner.py +++ b/src/nte_history_exporter/live_capture/runner.py @@ -24,6 +24,10 @@ EXPORT_PREFIXES = { "arc_miracle_box": "Arc", } +CAPTURE_SOURCE_LABELS = { + "windows_raw": "windows_packet", +} + def copy_to_clipboard(text: str) -> bool: try: @@ -60,8 +64,9 @@ def run_live_capture( *, interface_ip: str | None = None, capture_backend: str = "auto", - copy_clipboard: bool = True, + copy_clipboard: bool = False, write_debug_csv: bool = False, + user_uid: str | None = None, ) -> dict: local_ip = interface_ip or detect_local_ipv4() session = LiveHistorySession(local_ip) @@ -88,6 +93,7 @@ def run_live_capture( src_port=packet.src_port, dst_port=packet.dst_port, payload=packet.payload, + protocol=packet.protocol, ) ) if matched: @@ -130,6 +136,10 @@ def run_live_capture( ) exports = [] + resolved_user_uid = user_uid or session.user_uid + if session.kinds_seen() and not resolved_user_uid: + resolved_user_uid = console.prompt_user_uid() + capture_source = CAPTURE_SOURCE_LABELS.get(capture.name, capture.name) for kind in session.kinds_seen(): pairs = session.pairs_for_kind(kind) best_run, run_warnings = select_continuous_run_from_page_1(pairs) @@ -138,13 +148,15 @@ def run_live_capture( rows = annotate_groups(rows) warnings = run_warnings pages_seen = [p[0] for p in best_run] - csv_path, json_path = export_paths(kind) + csv_path, json_path = export_paths(kind, resolved_user_uid) if write_debug_csv: write_csv(csv_path, rows) export = build_export_json( rows, warnings, source="live_capture", + capture_source=capture_source, + user_uid=resolved_user_uid, pages_seen=pages_seen, ) payload = json.dumps(export, ensure_ascii=False, indent=2) @@ -162,9 +174,9 @@ def run_live_capture( console.print_results_header() if not exports: console.print_problem("No history pages were captured.") - console.print_note("This tool must already be running when you press Start on the") - console.print_note("game's main menu. Log out to the main menu, enter the game") - console.print_note("again, then reopen the history screen.") + console.print_note("Make sure the capture backend is running, then reopen the") + console.print_note("history screen and scroll from page 1. If no page messages") + console.print_note("appear, return to the main menu and re-enter the game.") return {"exports": []} for item in exports: @@ -189,24 +201,33 @@ def run_live_capture( console.print_success("Export copied to clipboard - paste it straight into your tracker.") else: console.print_note("Clipboard tool unavailable; use the JSON file shown above.") - elif len(exports) > 1: + elif copy_clipboard and len(exports) > 1: console.print_note("Multiple banners captured; clipboard copy skipped so one export") console.print_note("does not overwrite another.") return {"exports": exports} -def export_paths(kind: str) -> tuple[Path, Path]: +def export_paths(kind: str, user_uid: str | None = None) -> tuple[Path, Path]: export_dir = Path("exports") export_dir.mkdir(parents=True, exist_ok=True) prefix = EXPORT_PREFIXES.get(kind, "History") stamp = datetime.now().strftime("%Y%m%d_%H%M%S") - base = export_dir / f"{prefix}_{stamp}" + uid_prefix = _safe_filename_part(user_uid) if user_uid else "unknown" + base_name = f"{uid_prefix}_{prefix}_{stamp}" + base = export_dir / base_name csv_path = base.with_suffix(".csv") json_path = base.with_suffix(".json") counter = 2 while csv_path.exists() or json_path.exists(): - base = export_dir / f"{prefix}_{stamp}_{counter}" + base = export_dir / f"{base_name}_{counter}" csv_path = base.with_suffix(".csv") json_path = base.with_suffix(".json") counter += 1 return csv_path, json_path + + +def _safe_filename_part(value: str | None) -> str: + if not value: + return "unknown" + cleaned = "".join(ch for ch in value.strip() if ch.isalnum() or ch in ("-", "_")) + return cleaned or "unknown" diff --git a/src/nte_history_exporter/live_capture/session.py b/src/nte_history_exporter/live_capture/session.py index a745f94..e793294 100644 --- a/src/nte_history_exporter/live_capture/session.py +++ b/src/nte_history_exporter/live_capture/session.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections import deque +from collections import Counter, deque from dataclasses import dataclass from typing import Any @@ -19,6 +19,7 @@ from nte_history_exporter.decoder.protocol import ( request_page, ) from nte_history_exporter.decoder.run import build_rows_from_pairs +from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates @dataclass @@ -29,6 +30,7 @@ class UdpPacket: src_port: int dst_port: int payload: bytes + protocol: str = "udp" @dataclass @@ -57,6 +59,8 @@ class LiveHistorySession: self.last_capture_was_replacement = False self.requested_pages: dict[str, set[int]] = {} self.unanswered_pages: dict[str, dict[int, str]] = {} + self.user_uid: str | None = None + self.user_uid_candidates: Counter[str] = Counter() def _mark_unanswered(self, request: PendingRequest) -> None: if request.response_candidates: @@ -93,6 +97,12 @@ class LiveHistorySession: def process_packet(self, packet: UdpPacket) -> bool: self.packet_count += 1 + candidates = extract_user_uid_candidates(packet.payload) + if candidates: + self.user_uid_candidates.update(candidates) + self.user_uid = self.user_uid_candidates.most_common(1)[0][0] + if packet.protocol != "udp": + return False if packet.src_ip == self.local_ip and is_history_request(packet.payload): offset = int.from_bytes(packet.payload[31:35], "little") diff --git a/src/nte_history_exporter/live_capture/windows_raw.py b/src/nte_history_exporter/live_capture/windows_raw.py index 7c520f8..83b366b 100644 --- a/src/nte_history_exporter/live_capture/windows_raw.py +++ b/src/nte_history_exporter/live_capture/windows_raw.py @@ -10,12 +10,16 @@ RECEIVE_BUFFER_SIZE = 4 * 1024 * 1024 @dataclass -class ParsedIpUdpPacket: +class ParsedIpPacket: src_ip: str dst_ip: str src_port: int dst_port: int payload: bytes + protocol: str = "udp" + + +ParsedIpUdpPacket = ParsedIpPacket def detect_local_ipv4() -> str: @@ -42,24 +46,41 @@ def detect_local_ipv4() -> str: return candidates[0] -def parse_ipv4_udp_packet(data: bytes) -> ParsedIpUdpPacket | None: +def parse_ipv4_packet(data: bytes) -> ParsedIpPacket | None: if len(data) < 28: return None version_ihl = data[0] if version_ihl >> 4 != 4: return None ihl = (version_ihl & 0x0F) * 4 + total_length = struct.unpack_from("!H", data, 2)[0] + if total_length > 0: + data = data[:total_length] if len(data) < ihl + 8: return None protocol = data[9] - if protocol != 17: - return None src_ip = socket.inet_ntoa(data[12:16]) dst_ip = socket.inet_ntoa(data[16:20]) - src_port, dst_port, udp_len, _checksum = struct.unpack_from("!HHHH", data, ihl) - payload = data[ihl + 8 : ihl + udp_len] - return ParsedIpUdpPacket(src_ip, dst_ip, src_port, dst_port, payload) + if protocol == 17: + src_port, dst_port, udp_len, _checksum = struct.unpack_from("!HHHH", data, ihl) + payload = data[ihl + 8 : ihl + udp_len] + return ParsedIpPacket(src_ip, dst_ip, src_port, dst_port, payload, "udp") + if protocol == 6: + if len(data) < ihl + 20: + return None + src_port, dst_port = struct.unpack_from("!HH", data, ihl) + tcp_header_len = (data[ihl + 12] >> 4) * 4 + if tcp_header_len < 20 or len(data) < ihl + tcp_header_len: + return None + payload = data[ihl + tcp_header_len :] + return ParsedIpPacket(src_ip, dst_ip, src_port, dst_port, payload, "tcp") + return None + + +def parse_ipv4_udp_packet(data: bytes) -> ParsedIpPacket | None: + packet = parse_ipv4_packet(data) + return packet if packet and packet.protocol == "udp" else None def open_raw_udp_socket(local_ip: str) -> socket.socket: @@ -79,7 +100,7 @@ def read_packets(sock: socket.socket): except socket.timeout: yield None continue - packet = parse_ipv4_udp_packet(data) + packet = parse_ipv4_packet(data) if packet is None: continue yield packet diff --git a/tests/test_boundary_export.py b/tests/test_boundary_export.py index 4a2b24b..ac1a184 100644 --- a/tests/test_boundary_export.py +++ b/tests/test_boundary_export.py @@ -20,6 +20,7 @@ from nte_history_exporter.decoder.protocol import decode_response_records, histo from nte_history_exporter.constants import POOL_META from nte_history_exporter.mappings import ARC_META, CHARACTERS, ITEMS, REWARDS_BY_ID from nte_history_exporter.decoder.protocol import decode_reward_key, infer_reward_type +from nte_history_exporter.decoder.user_uid import extract_user_uid from nte_history_exporter.decoder.arc import ( arc_request_page, build_arc_rows_from_pairs, @@ -39,8 +40,10 @@ from nte_history_exporter.live_capture.libpcap import ( _extract_ipv4_frame, LibpcapUnavailable, ) +from nte_history_exporter.live_capture.windows_raw import parse_ipv4_packet from nte_history_exporter.live_capture.backends import open_capture_backend from nte_history_exporter.export.json_export import build_export_json +from nte_history_exporter.live_capture.runner import export_paths from nte_history_exporter.pool_mappings import load_pool_mappings, pool_meta_from_mapping @@ -347,10 +350,94 @@ class BoundaryExportTests(unittest.TestCase): self.assertEqual(export["format"], "nte-history-export") self.assertIn("exporter", export) + self.assertNotIn("user_uid", export) self.assertNotIn("record_hex", export["records"][0]) self.assertNotIn("request_msg", export["records"][0]) self.assertNotIn("response_msg", export["records"][0]) + def test_export_includes_user_uid_when_provided(self): + rows = load_reference_csv("monopoly_history_poc_13_pages_1_to_5_v4.csv") + annotated = annotate_groups(rows) + export = build_export_json( + annotated, + [], + capture_source="npcap", + user_uid="123456789", + ) + + self.assertEqual(list(export).index("user_uid"), list(export).index("records") - 1) + self.assertEqual(export["capture_source"], "npcap") + self.assertEqual(export["user_uid"], "123456789") + + def test_export_paths_include_user_uid_banner_and_timestamp(self): + _csv_path, json_path = export_paths("limited_character", "218216016349") + + self.assertRegex( + json_path.name, + r"^218216016349_Limited_\d{8}_\d{6}(?:_\d+)?\.json$", + ) + + def test_extracts_user_uid_from_record_context(self): + payload = ( + b"\x00" * 24 + + (218216016349).to_bytes(8, "little") + + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00" + ) + + self.assertEqual(extract_user_uid(payload), "218216016349") + + def test_extracts_user_uid_from_private_spawn_record_context(self): + payload = ( + b"\x88\x00\x00\x00\x10\x00\x00\x00" + + (218216016349).to_bytes(8, "little") + + b"\x08\x00\x0c\x00\x07\x00\x08\x00\x08\x00\x00\x00" + + b"\x00\x00\x00\x01\x08\x00\x00\x00\x04\x00\x04\x00" + + b"\x04\x00\x00\x00\x16\x00\x00\x00PrivateSpawnInfoRecord\x00" + ) + + self.assertEqual(extract_user_uid(payload), "218216016349") + + def test_does_not_extract_user_uid_from_wrong_record_offset(self): + payload = ( + b"\x00" * 28 + + (218216016349).to_bytes(8, "little") + + b"\x00\x00\x00\x00TagOthers\x00" + ) + + self.assertIsNone(extract_user_uid(payload)) + + def test_does_not_extract_old_eight_digit_false_positive_as_user_uid(self): + payload = ( + b"WholeVehicleData\x00\x00\x00\x00\x00o<\x00\x00\x05\x00\x00\x00" + b"\x0b\x00\x00\x00Vehicle015\x00\x0b\x00\x00\x00buyvehicle\x00" + b"\x09\x00\x00\x0015363624\x00\x06\x00\x00\x00" + ) + + self.assertIsNone(extract_user_uid(payload)) + + def test_ipv4_parser_extracts_tcp_payload_for_user_uid_detection(self): + payload = ( + (218216016349).to_bytes(8, "little") + + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00" + ) + tcp_header = bytearray(20) + tcp_header[0:2] = (40000).to_bytes(2, "big") + tcp_header[2:4] = (30000).to_bytes(2, "big") + tcp_header[12] = 5 << 4 + total_len = 20 + len(tcp_header) + len(payload) + ip_header = bytearray(20) + ip_header[0] = 0x45 + ip_header[2:4] = total_len.to_bytes(2, "big") + ip_header[9] = 6 + ip_header[12:16] = bytes([192, 0, 2, 1]) + ip_header[16:20] = bytes([198, 51, 100, 2]) + + packet = parse_ipv4_packet(bytes(ip_header) + bytes(tcp_header) + payload) + + self.assertIsNotNone(packet) + self.assertEqual(packet.protocol, "tcp") + self.assertEqual(packet.payload, payload) + def test_limited_selector_and_marker_decode(self): request = bytearray(45) request[31:35] = (4).to_bytes(4, "little") From fefe3a1567f2aa3c043fbdfd0703a545a8cfdbd7 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Wed, 17 Jun 2026 12:05:27 +0100 Subject: [PATCH 2/3] Dedupe missing-page notices and colorize output Add brighter white ANSI constant and update console messages to use consistent color/styling. Simplify print_missing_pages (remove per-page reasons) and make the recovery instruction more explicit. In the live capture runner introduce active_gap_notices to suppress repeated missing-page alerts per pool kind and clear the flag when the gap is recovered. These changes reduce duplicate noise and improve visibility of the guidance shown to users. --- src/nte_history_exporter/console.py | 18 +++++++++++------- .../live_capture/runner.py | 11 +++++------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/nte_history_exporter/console.py b/src/nte_history_exporter/console.py index c92fb41..40042c9 100644 --- a/src/nte_history_exporter/console.py +++ b/src/nte_history_exporter/console.py @@ -13,6 +13,7 @@ DIM = "\x1b[2m" CYAN = "\x1b[36m" GREEN = "\x1b[32m" YELLOW = "\x1b[33m" +BRIGHT_WHITE = "\x1b[97m" _ansi: bool | None = None @@ -91,17 +92,20 @@ def print_page_captured(label: str, page: int | None, *, recaptured: bool = Fals print(style(" + ", GREEN, BOLD) + label + style(f" {action} {page}", DIM)) -def print_missing_pages(label: str, pages: list[int], reasons: dict[int, str] | None = None) -> None: +def print_missing_pages(label: str, pages: list[int]) -> None: page_list = ", ".join(str(page) for page in pages) - print(style(" ! ", YELLOW, BOLD) + f"{label} missing page(s): {page_list}.") - if reasons: - for page in pages: - print(style(f" Page {page}: {reasons[page]}.", DIM)) - print(" Close and reopen this history board, then scroll down again.") + print(style(" ! ", YELLOW, BOLD) + style(f"{label} missing page(s): {page_list}.", YELLOW, BOLD)) + print( + style( + " Close and reopen this history board, then scroll down until page gap recovered is shown.", + BRIGHT_WHITE, + BOLD, + ) + ) def print_page_gap_recovered(label: str) -> None: - print(style(" + ", GREEN, BOLD) + f"{label} page gap recovered.") + print(style(" + ", GREEN, BOLD) + style(f"{label} page gap recovered.", GREEN, BOLD)) def print_capture_stats(received: int, dropped: int, interface_dropped: int) -> None: diff --git a/src/nte_history_exporter/live_capture/runner.py b/src/nte_history_exporter/live_capture/runner.py index 7763867..3f41f50 100644 --- a/src/nte_history_exporter/live_capture/runner.py +++ b/src/nte_history_exporter/live_capture/runner.py @@ -76,6 +76,7 @@ def run_live_capture( if capture.fallback_reason: console.print_capture_fallback(capture.fallback_reason) reported_missing_pages: dict[str, tuple[int, ...]] = {} + active_gap_notices: set[str] = set() try: with StopKeyMonitor() as stop_key: @@ -115,14 +116,12 @@ def run_live_capture( label = POOL_META.get(kind, POOL_META["permanent"])["name"] missing_pages = tuple(session.missing_pages(kind)) previously_missing = reported_missing_pages.get(kind, ()) - if missing_pages and missing_pages != previously_missing: - reasons = { - page: session.missing_page_reason(kind, page) - for page in missing_pages - } - console.print_missing_pages(label, list(missing_pages), reasons) + if missing_pages and kind not in active_gap_notices: + console.print_missing_pages(label, list(missing_pages)) + active_gap_notices.add(kind) elif previously_missing and not missing_pages: console.print_page_gap_recovered(label) + active_gap_notices.discard(kind) reported_missing_pages[kind] = missing_pages finally: stats = capture.stats() From c5373aa9e01d0bc84ef4ef32f2cd7f121adfacdb Mon Sep 17 00:00:00 2001 From: Golumpa Date: Wed, 17 Jun 2026 14:54:25 +0100 Subject: [PATCH 3/3] Add PyInstaller release workflow --- .github/workflows/release.yml | 166 ++++++++++++++++++ docs/packaging.md | 45 +++++ packaging/NTE History Exporter.spec | 47 +++++ pyproject.toml | 3 + src/nte_history_exporter/cli.py | 5 +- .../live_capture/libpcap.py | 3 +- src/nte_history_exporter/mappings.py | 3 +- tests/test_boundary_export.py | 24 +++ tools/build-windows.ps1 | 27 +++ 9 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 docs/packaging.md create mode 100644 packaging/NTE History Exporter.spec create mode 100644 tools/build-windows.ps1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9e29a1a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,166 @@ +name: Release + +on: + workflow_dispatch: + push: + branches: [main] + +permissions: + contents: write + +jobs: + build-windows: + name: Build Windows + if: github.ref == 'refs/heads/main' + runs-on: windows-latest + + steps: + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Get Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[build]" + + - name: Test + run: python -m unittest discover -s tests + + - name: Build + run: python -m PyInstaller --clean --noconfirm "packaging/NTE History Exporter.spec" + + - name: Smoke Test + run: .\dist\nte-history-exporter.exe --help + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: windows + path: dist/nte-history-exporter.exe + + build-linux: + name: Build Linux + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + + steps: + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Install libpcap + run: sudo apt-get update && sudo apt-get install -y libpcap-dev + + - name: Get Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[build]" + + - name: Test + run: python -m unittest discover -s tests + + - name: Build + run: python -m PyInstaller --clean --noconfirm "packaging/NTE History Exporter.spec" + + - name: Smoke Test + run: ./dist/nte-history-exporter --help + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: linux + path: dist/nte-history-exporter + + build-macos: + name: Build Macos + if: github.ref == 'refs/heads/main' + runs-on: macos-15-intel + + steps: + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Install libpcap + run: brew update && brew install libpcap + + - name: Get Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[build]" + + - name: Test + run: python -m unittest discover -s tests + + - name: Build + run: python -m PyInstaller --clean --noconfirm "packaging/NTE History Exporter.spec" + + - name: Smoke Test + run: ./dist/nte-history-exporter --help + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: macos + path: dist/nte-history-exporter + + release: + name: Release + if: github.ref == 'refs/heads/main' + needs: [build-windows, build-linux, build-macos] + runs-on: ubuntu-latest + + steps: + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Get package version + id: package-version + run: | + version="$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Download Artifacts + uses: actions/download-artifact@v4 + + - name: Rename + run: | + version="${{ steps.package-version.outputs.version }}" + cp windows/nte-history-exporter.exe nte-history-exporter.exe + zip nte-history-exporter-v${version}-x86_64-pc-windows-msvc.zip nte-history-exporter.exe + cp linux/nte-history-exporter nte-history-exporter + chmod +x nte-history-exporter + cp nte-history-exporter nte-history-exporter-linux + zip nte-history-exporter-v${version}-x86_64-unknown-linux-gnu.zip nte-history-exporter + cp macos/nte-history-exporter nte-history-exporter + chmod +x nte-history-exporter + cp nte-history-exporter nte-history-exporter-macos + zip nte-history-exporter-v${version}-x86_64-apple-darwin.zip nte-history-exporter + + - name: Release + uses: marvinpinto/action-automatic-releases@latest + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + automatic_release_tag: v${{ steps.package-version.outputs.version }} + prerelease: false + title: v${{ steps.package-version.outputs.version }} Release + files: | + nte-history-exporter.exe + nte-history-exporter-linux + nte-history-exporter-macos + nte-history-exporter-v${{ steps.package-version.outputs.version }}-x86_64-pc-windows-msvc.zip + nte-history-exporter-v${{ steps.package-version.outputs.version }}-x86_64-unknown-linux-gnu.zip + nte-history-exporter-v${{ steps.package-version.outputs.version }}-x86_64-apple-darwin.zip diff --git a/docs/packaging.md b/docs/packaging.md new file mode 100644 index 0000000..b3baac3 --- /dev/null +++ b/docs/packaging.md @@ -0,0 +1,45 @@ +# Packaging + +## Windows local build + +Build the Windows console executable with: + +```powershell +.\tools\build-windows.ps1 +``` + +The output is: + +```text +dist\nte-history-exporter.exe +``` + +Run live capture from an elevated PowerShell prompt: + +```powershell +.\dist\nte-history-exporter.exe --live +``` + +## Packet capture runtime + +Windows live capture defaults to `--capture-backend auto`, which tries an installed system Npcap runtime first, then the built-in raw-socket backend. If `--capture-backend libpcap` is specified, raw fallback is disabled. If `--capture-backend raw` is specified, only the Windows raw-socket backend is used. + +Npcap is Windows-only; Linux and macOS use the system libpcap runtime. + +The GitHub Actions release does not download or bundle Npcap. The Windows executable can use a user's installed Npcap runtime when present, then falls back to raw sockets. A future pktmon backend can provide the same kind of no-extra-install Windows path that stardb-exporter uses. + +Npcap is not committed or bundled by default. Windows users who want the Npcap backend should install Npcap separately. Users who do not install Npcap can still use the Windows raw-socket fallback. + +## Cross-platform CI shape + +Use one GitHub Actions job per OS. PyInstaller must build on the target platform, so Windows produces `.exe`, macOS produces a macOS binary, and Linux produces a Linux binary. + +This repository has a manual release workflow at: + +```text +.github\workflows\release.yml +``` + +It builds Windows, Linux, and macOS artifacts, then publishes a `v` GitHub release with raw binaries and versioned zip files. + +Linux runners install the libpcap runtime for parity with live-capture usage, but the build itself uses `ctypes` and does not link against libpcap at package time. diff --git a/packaging/NTE History Exporter.spec b/packaging/NTE History Exporter.spec new file mode 100644 index 0000000..3797db2 --- /dev/null +++ b/packaging/NTE History Exporter.spec @@ -0,0 +1,47 @@ +# -*- mode: python ; coding: utf-8 -*- + +from pathlib import Path + + +block_cipher = None +root = Path(SPECPATH).parent + + +a = Analysis( + [str(root / "src" / "nte_history_exporter" / "cli.py")], + pathex=[str(root / "src")], + binaries=[], + datas=[(str(root / "mappings"), "mappings")], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name="nte-history-exporter", + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/pyproject.toml b/pyproject.toml index d9f6034..24e2f3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,9 @@ description = "Cross-platform Neverness to Everness pull-history exporter." requires-python = ">=3.10" dependencies = [] +[project.optional-dependencies] +build = ["pyinstaller>=6.0"] + [project.scripts] nte-history-exporter = "nte_history_exporter.cli:main" diff --git a/src/nte_history_exporter/cli.py b/src/nte_history_exporter/cli.py index 426b564..be9a8e2 100644 --- a/src/nte_history_exporter/cli.py +++ b/src/nte_history_exporter/cli.py @@ -24,7 +24,10 @@ def build_parser() -> argparse.ArgumentParser: "--capture-backend", choices=["auto", "libpcap", "raw"], default="auto", - help="live capture backend; auto prefers Npcap/libpcap and falls back to raw sockets on Windows", + help=( + "live capture backend; auto prefers Npcap/libpcap and falls back to raw sockets on Windows " + "(default: %(default)s)" + ), ) parser.add_argument("--copy-clipboard", action="store_true", help="copy a single live export to clipboard") parser.add_argument("--debug", action="store_true", help="also write research CSVs next to the JSON exports") diff --git a/src/nte_history_exporter/live_capture/libpcap.py b/src/nte_history_exporter/live_capture/libpcap.py index ba78f06..1ee2acf 100644 --- a/src/nte_history_exporter/live_capture/libpcap.py +++ b/src/nte_history_exporter/live_capture/libpcap.py @@ -103,7 +103,8 @@ def _load_library() -> ctypes.CDLL: npcap_dir = _windows_npcap_directory() if npcap_dir.is_dir() and hasattr(os, "add_dll_directory"): dll_directory = os.add_dll_directory(str(npcap_dir)) - candidates.extend([str(npcap_dir / "wpcap.dll"), "wpcap.dll"]) + candidates.append(str(npcap_dir / "wpcap.dll")) + candidates.append("wpcap.dll") elif sys.platform == "darwin": candidates.extend( [ diff --git a/src/nte_history_exporter/mappings.py b/src/nte_history_exporter/mappings.py index 2ecec43..27213a1 100644 --- a/src/nte_history_exporter/mappings.py +++ b/src/nte_history_exporter/mappings.py @@ -1,10 +1,11 @@ from __future__ import annotations import json +import sys from pathlib import Path from typing import Any -PROJECT_ROOT = Path(__file__).resolve().parents[2] +PROJECT_ROOT = Path(getattr(sys, "_MEIPASS", Path(__file__).resolve().parents[2])) MAPPINGS_DIR = PROJECT_ROOT / "mappings" diff --git a/tests/test_boundary_export.py b/tests/test_boundary_export.py index ac1a184..4ff0cb1 100644 --- a/tests/test_boundary_export.py +++ b/tests/test_boundary_export.py @@ -38,6 +38,7 @@ from nte_history_exporter.live_capture.libpcap import ( DLT_LOOP, DLT_RAW, _extract_ipv4_frame, + _load_library, LibpcapUnavailable, ) from nte_history_exporter.live_capture.windows_raw import parse_ipv4_packet @@ -138,6 +139,29 @@ class BoundaryExportTests(unittest.TestCase): raw_capture.assert_not_called() + def test_windows_libpcap_load_prefers_system_then_path(self): + system_dir = Path("C:/Windows/System32/Npcap") + loaded = Mock() + with ( + patch("nte_history_exporter.live_capture.libpcap.sys.platform", "win32"), + patch("nte_history_exporter.live_capture.libpcap._windows_npcap_directory", return_value=system_dir), + patch("nte_history_exporter.live_capture.libpcap.Path.is_dir", return_value=True), + patch("nte_history_exporter.live_capture.libpcap.os.add_dll_directory", return_value=Mock()), + patch( + "nte_history_exporter.live_capture.libpcap.ctypes.CDLL", + side_effect=[OSError("system missing"), loaded], + ) as cdll, + ): + self.assertIs(_load_library(), loaded) + + self.assertEqual( + [call.args[0] for call in cdll.call_args_list], + [ + str(system_dir / "wpcap.dll"), + "wpcap.dll", + ], + ) + def test_libpcap_link_layers_extract_ipv4_packets(self): ip_packet = bytes.fromhex("4500001c0000000040110000c0000201c6336402") + bytes(8) ethernet = bytes(12) + bytes.fromhex("0800") + ip_packet diff --git a/tools/build-windows.ps1 b/tools/build-windows.ps1 new file mode 100644 index 0000000..44e6225 --- /dev/null +++ b/tools/build-windows.ps1 @@ -0,0 +1,27 @@ +$ErrorActionPreference = 'Stop' + +Set-Location (Split-Path -Parent $PSScriptRoot) + +$python = Join-Path $PWD '.venv\Scripts\python.exe' +if (-not (Test-Path $python)) { + python -m venv .venv +} + +& $python -m pip install --upgrade pip +& $python -m pip install -e '.[build]' + +& $python -m PyInstaller --clean --noconfirm 'packaging\NTE History Exporter.spec' +if ($LASTEXITCODE -ne 0) { + throw "PyInstaller failed with exit code $LASTEXITCODE" +} + +$exe = Join-Path $PWD 'dist\nte-history-exporter.exe' +if (-not (Test-Path $exe)) { + throw "Expected build output was not created: $exe" +} + +Write-Host "Built $exe" +& $exe --help +if ($LASTEXITCODE -ne 0) { + throw "Built executable smoke test failed with exit code $LASTEXITCODE" +}