0.1.1 - Add libpcap/Npcap backend and robust live capture

Introduce a cross-platform libpcap/Npcap capture backend with a Windows raw-socket fallback and plumbing to select backends via --capture-backend. Add ctypes-based libpcap wrapper (open_libpcap_capture, CaptureStats, LibpcapUnavailable) and a backend factory; refactor live capture runner to use the new backend, StopKeyMonitor, and improved clipboard handling. Make session and protocol decoders resilient to pipelined/multi-page responses and non-byte-aligned streams (alignment iterator, alignment-aware decoding, embedded-record trimming, slice support). Update mitmproxy flows adapter to reuse LiveHistorySession. Expand console messages and docs/README to explain cross-platform requirements, usage, and capture diagnostics. Minor: update package description in pyproject and adjust ARC/response handling and row bookkeeping to record row indices.
This commit is contained in:
Golumpa 2026-06-12 12:58:15 +01:00
parent 1144e33a57
commit 0e78906e3a
20 changed files with 1234 additions and 147 deletions

View file

@ -1 +1 @@
__version__ = "0.1.0"
__version__ = "0.1.1"

View file

@ -1,24 +1,12 @@
from __future__ import annotations
import struct
from pathlib import Path
from typing import Any
from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
from nte_history_exporter.decoder.arc import (
arc_request_page,
build_arc_rows_from_pairs,
is_arc_history_request,
parse_arc_response,
select_continuous_arc_run,
)
from nte_history_exporter.decoder.protocol import (
history_request_kind,
is_history_request,
request_page,
response_contains_history_marker,
)
from nte_history_exporter.decoder.run import build_rows_from_pairs, fmt_packet_time
from nte_history_exporter.decoder.arc import build_arc_rows_from_pairs, select_continuous_arc_run
from nte_history_exporter.decoder.run import build_rows_from_pairs
from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket
def parse_tnetstring(data: bytes, i: int = 0) -> tuple[Any, int]:
@ -87,47 +75,26 @@ def find_udp_flow(flows: list[Any], preferred_index: int | None = None) -> tuple
return idx, flow
def pair_response(messages: list[Any], request_index: int) -> tuple[int | None, bytes, float | None]:
for j in range(request_index + 1, min(request_index + 50, len(messages))):
from_client, content, ts = messages[j]
if not from_client and len(content) >= 150 and response_contains_history_marker(content):
return j, content, ts
return None, b"", None
def pair_arc_response(messages: list[Any], request_index: int) -> tuple[int | None, bytes, float | None]:
for j in range(request_index + 1, min(request_index + 30, len(messages))):
from_client, content, ts = messages[j]
if not from_client and len(content) >= 100 and parse_arc_response(content):
return j, content, ts
return None, b"", None
def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> dict[str, Any]:
flows = read_flows(path)
resolved_flow_index, flow = find_udp_flow(flows, flow_index)
messages = flow[b"messages"]
pairs = []
arc_pairs = []
for i, msg in enumerate(messages):
local_ip = "192.0.2.1"
remote_ip = "198.51.100.1"
local_port = 50000
remote_port = 40000
session = LiveHistorySession(local_ip)
for msg in messages:
from_client, content, ts = msg
if from_client and is_arc_history_request(content):
page = arc_request_page(content)
response_index, response_content, response_ts = pair_arc_response(messages, i)
if response_index is not None:
arc_pairs.append((page, page * 2, i, ts, response_index, response_ts, response_content))
continue
if not from_client or not is_history_request(content):
continue
kind = history_request_kind(content)
offset = struct.unpack_from("<I", content, 31)[0]
page = request_page(content)
response_index, response_content, response_ts = pair_response(messages, i)
if response_index is not None:
pairs.append((page, offset, i, ts, response_index, response_ts, response_content, kind))
if from_client:
packet = UdpPacket(ts, local_ip, remote_ip, local_port, remote_port, content)
else:
packet = UdpPacket(ts, remote_ip, local_ip, remote_port, local_port, content)
session.process_packet(packet)
pairs = [pair for pair in session.pairs if pair[7] != "arc_miracle_box"]
arc_pairs = [pair for pair in session.pairs if pair[7] == "arc_miracle_box"]
best_run, run_warnings = select_continuous_run_from_page_1(pairs)
rows_out = build_rows_from_pairs(best_run)
best_arc_run, arc_warnings = select_continuous_arc_run(arc_pairs)

View file

@ -8,17 +8,24 @@ from nte_history_exporter.adapters.mitmproxy_flows import decode_mitmproxy_flows
from nte_history_exporter.decoder.boundary import annotate_groups
from nte_history_exporter.export.csv_export import write_csv
from nte_history_exporter.export.json_export import build_export_json
from nte_history_exporter.live_capture.libpcap import LibpcapUnavailable
from nte_history_exporter.live_capture.runner import export_paths, run_live_capture
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Decode NTE Monopoly history from mitmproxy .flows captures or live capture."
description="Decode NTE pull history from mitmproxy .flows captures or live capture."
)
parser.add_argument("capture_source", nargs="?", help="mitmproxy .flows file; omit when using --live")
parser.add_argument("--live", action="store_true", help="capture live UDP traffic instead of reading a .flows file")
parser.add_argument("--flow-index", type=int, default=None)
parser.add_argument("--interface-ip", default=None, help="local IPv4 address to bind for live capture")
parser.add_argument(
"--capture-backend",
choices=["auto", "libpcap", "raw"],
default="auto",
help="live capture backend; auto prefers Npcap/libpcap and falls back to raw sockets on Windows",
)
parser.add_argument("--no-clipboard", action="store_true", help="do not copy live exports to clipboard")
parser.add_argument("--debug", action="store_true", help="also write research CSVs next to the JSON exports")
return parser
@ -28,12 +35,18 @@ def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
console.print_banner()
if args.live or not args.capture_source:
run_live_capture(
interface_ip=args.interface_ip,
copy_clipboard=not args.no_clipboard,
write_debug_csv=args.debug,
)
return 0
try:
run_live_capture(
interface_ip=args.interface_ip,
capture_backend=args.capture_backend,
copy_clipboard=not args.no_clipboard,
write_debug_csv=args.debug,
)
return 0
except (LibpcapUnavailable, PermissionError) as exc:
console.print_problem(str(exc))
console.print_note("Install/enable Npcap or libpcap and run with packet-capture permissions.")
return 1
decoded = decode_mitmproxy_flows(args.capture_source, args.flow_index)
if decoded["arc_rows"] and not decoded["rows"]:

View file

@ -61,9 +61,11 @@ def print_banner() -> None:
print(rule("="))
def print_live_instructions(local_ip: str) -> None:
def print_live_instructions(local_ip: str, backend: str = "windows_raw", detail: str = "") -> None:
print()
print(style(" Listening on ", DIM) + style(local_ip, BOLD))
backend_detail = f" ({detail})" if detail and detail != local_ip else ""
print(style(f" Capture backend: {backend}{backend_detail}", DIM))
print()
print(style(" How to export your pull history", BOLD))
print(" 1. This tool must be running BEFORE you press Start on")
@ -84,8 +86,38 @@ def print_live_instructions(local_ip: str) -> None:
print(rule())
def print_page_captured(label: str, page: int | None) -> None:
print(style(" + ", GREEN, BOLD) + label + style(f" page {page}", DIM))
def print_page_captured(label: str, page: int | None, *, recaptured: bool = False) -> None:
action = "recaptured" if recaptured else "page"
print(style(" + ", GREEN, BOLD) + label + style(f" {action} {page}", DIM))
def print_missing_pages(label: str, pages: list[int], reasons: dict[int, str] | None = None) -> None:
page_list = ", ".join(str(page) for page in pages)
print(style(" ! ", YELLOW, BOLD) + f"{label} missing page(s): {page_list}.")
if reasons:
for page in pages:
print(style(f" Page {page}: {reasons[page]}.", DIM))
print(" Close and reopen this history board, then scroll down again.")
def print_page_gap_recovered(label: str) -> None:
print(style(" + ", GREEN, BOLD) + f"{label} page gap recovered.")
def print_capture_stats(received: int, dropped: int, interface_dropped: int) -> None:
text = (
f"Capture stats: processed {received}, buffer dropped {dropped}, "
f"interface dropped {interface_dropped}"
)
if dropped or interface_dropped:
print(style(f" ! {text}", YELLOW, BOLD))
else:
print(style(f" {text}", DIM))
def print_capture_fallback(reason: str) -> None:
print(style(" ! Npcap unavailable; using Windows raw capture.", YELLOW))
print(style(f" {reason}", DIM))
def print_results_header() -> None:

View file

@ -11,7 +11,7 @@ LIMITED_CHARACTER_BANNER_NAME = "Limited Character Board"
ARC_BANNER_ID = "Arc_MiracleBox"
ARC_BANNER_NAME = "Arc Miracle Box"
EXPORTER_NAME = "nte-history-exporter"
EXPORTER_VERSION = "0.1.0"
EXPORTER_VERSION = "0.1.1"
HISTORY_REQUEST_BANNER = 4220
HISTORY_REQUEST_LENGTH = 45

View file

@ -25,7 +25,7 @@ from nte_history_exporter.mappings import ARC_META
def is_arc_history_request(content: bytes) -> bool:
return len(content) == ARC_HISTORY_REQUEST_LENGTH and struct.unpack_from("<I", content, 24)[0] == ARC_HISTORY_REQUEST_BANNER
return len(content) >= ARC_HISTORY_REQUEST_LENGTH and struct.unpack_from("<I", content, 24)[0] == ARC_HISTORY_REQUEST_BANNER
def arc_request_page(content: bytes) -> int:
@ -122,6 +122,9 @@ def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
for pair in pairs:
page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7]
records = parse_arc_response(response)
if len(pair) > 9:
slice_start, slice_count = pair[8:10]
records = records[slice_start : slice_start + slice_count]
for row_index, record in enumerate(records, start=1):
rows.append(
{

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import struct
from datetime import datetime, timezone
from typing import Any
from typing import Any, Iterator
from nte_history_exporter.constants import (
DOTNET_UNIX_EPOCH_SECONDS,
@ -73,7 +73,7 @@ def decode_history_timestamp(raw8: bytes) -> tuple[int, float, str]:
def history_request_kind(content: bytes) -> str:
if len(content) != HISTORY_REQUEST_LENGTH or struct.unpack_from("<I", content, 35)[0] != HISTORY_REQUEST_BANNER:
if len(content) < HISTORY_REQUEST_LENGTH or struct.unpack_from("<I", content, 35)[0] != HISTORY_REQUEST_BANNER:
return ""
selector = struct.unpack_from("<I", content, 40)[0]
if selector == PERMANENT_SELECTOR:
@ -92,7 +92,22 @@ def request_page(content: bytes) -> int:
def response_contains_history_marker(content: bytes) -> bool:
return any(marker in content for marker in MARKERS)
return any(
marker in candidate
for candidate in iter_history_response_alignments(content)
for marker in MARKERS
)
def iter_history_response_alignments(content: bytes) -> Iterator[bytes]:
yield content
for bit_offset in range(1, 8):
mask = (1 << bit_offset) - 1
yield bytes(
(content[index] >> bit_offset)
| ((content[index + 1] & mask) << (8 - bit_offset))
for index in range(len(content) - 1)
)
def extract_key(chunk_without_marker: bytes) -> str:
@ -161,7 +176,7 @@ def guess_quantity(chunk_hex: str, reward_id: str, result_type: str | None = Non
return None
def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
def _decode_aligned_response_records(response_content: bytes) -> list[dict[str, Any]]:
marker = b""
marker_offsets: list[int] = []
for candidate_marker in MARKERS:
@ -176,9 +191,41 @@ def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
prev = 0x50
for row_index, marker_offset in enumerate(marker_offsets, start=1):
record_start = prev
chunk = response_content[prev:marker_offset]
full_record = response_content[prev : marker_offset + len(marker) + 8]
dice, dice_raw, dice_offset = extract_dice(chunk)
if dice is None and len(chunk) > 32:
embedded_candidates = []
original_key = extract_key(chunk)
original_key_bytes = bytes.fromhex(original_key) if original_key else b""
for trim in range(1, min(96, len(chunk))):
candidate = chunk[trim:]
candidate_dice, candidate_raw, candidate_offset = extract_dice(candidate)
if (
candidate_dice is not None
and candidate_offset in (0, 5)
and extract_key(candidate) == original_key
):
key_count = candidate.count(original_key_bytes) if original_key_bytes else 0
key_position = candidate.find(original_key_bytes) if original_key_bytes else len(candidate)
embedded_candidates.append(
(
-key_count,
candidate_offset != 5,
key_position,
-trim,
trim,
candidate,
candidate_dice,
candidate_raw,
candidate_offset,
)
)
if embedded_candidates:
_, _, _, _, trim, chunk, dice, dice_raw, dice_offset = min(embedded_candidates)
record_start = prev + trim
full_record = response_content[prev + trim : marker_offset + len(marker) + 8]
result_type, result_source_raw = classify_result_type(chunk, dice, dice_offset)
if result_type == "points_gift":
dice = 0
@ -195,7 +242,7 @@ def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
rows.append(
{
"row": row_index,
"record_start": prev,
"record_start": record_start,
"record_end": marker_offset + len(marker) + 8,
"record_len": len(full_record),
"dice": dice,
@ -223,3 +270,16 @@ def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
)
prev = marker_offset + len(marker) + 8
return rows
def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
for candidate in iter_history_response_alignments(response_content):
if not any(marker in candidate for marker in MARKERS):
continue
try:
rows = _decode_aligned_response_records(candidate)
except (OSError, OverflowError, ValueError):
continue
if rows:
return rows
return []

View file

@ -20,6 +20,9 @@ def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
kind = pair[7] if len(pair) > 7 else "permanent"
pool = POOL_META.get(kind, POOL_META["permanent"])
records = decode_response_records(response_content)
if len(pair) > 9:
slice_start, slice_count = pair[8:10]
records = records[slice_start : slice_start + slice_count]
if not records:
rows_out.append(
{
@ -38,7 +41,7 @@ def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
}
)
continue
for record in records:
for row_index, record in enumerate(records, start=1):
rows_out.append(
{
"page": page,
@ -52,6 +55,7 @@ def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
"response_len": len(response_content),
"record_count": len(records),
**record,
"row": row_index,
}
)
return rows_out

View file

@ -0,0 +1,74 @@
from __future__ import annotations
import socket
import sys
from dataclasses import dataclass
from typing import Iterator, Protocol
from nte_history_exporter.live_capture.libpcap import (
CaptureStats,
LibpcapUnavailable,
open_libpcap_capture,
)
from nte_history_exporter.live_capture.windows_raw import (
ParsedIpUdpPacket,
open_raw_udp_socket,
read_packets,
)
class CaptureBackend(Protocol):
name: str
detail: str
fallback_reason: str
def packets(self) -> Iterator[ParsedIpUdpPacket | None]: ...
def stats(self) -> CaptureStats | None: ...
def close(self) -> None: ...
@dataclass
class RawSocketCapture:
local_ip: str
fallback_reason: str = ""
def __post_init__(self) -> None:
if sys.platform != "win32":
raise RuntimeError("the raw socket capture backend is only available on Windows")
self.name = "windows_raw"
self.detail = self.local_ip
self.socket = open_raw_udp_socket(self.local_ip)
def packets(self) -> Iterator[ParsedIpUdpPacket | None]:
return read_packets(self.socket)
def stats(self) -> CaptureStats | None:
return None
def close(self) -> None:
try:
self.socket.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF)
except Exception:
pass
self.socket.close()
def open_capture_backend(local_ip: str, requested: str = "auto") -> CaptureBackend:
if requested not in {"auto", "libpcap", "raw"}:
raise ValueError(f"unknown capture backend: {requested}")
if requested in {"auto", "libpcap"}:
try:
capture = open_libpcap_capture(local_ip)
except LibpcapUnavailable as exc:
if requested != "auto" or sys.platform != "win32":
raise
return RawSocketCapture(local_ip, fallback_reason=str(exc))
capture.name = "npcap" if sys.platform == "win32" else "libpcap"
capture.detail = capture.device
capture.fallback_reason = ""
return capture
return RawSocketCapture(local_ip)

View file

@ -0,0 +1,404 @@
from __future__ import annotations
import ctypes
import ctypes.util
import os
import socket
import struct
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Iterator
from nte_history_exporter.live_capture.windows_raw import ParsedIpUdpPacket, parse_ipv4_udp_packet
PCAP_ERRBUF_SIZE = 256
SNAP_LENGTH = 65535
CAPTURE_BUFFER_SIZE = 16 * 1024 * 1024
READ_TIMEOUT_MS = 100
DLT_NULL = 0
DLT_EN10MB = 1
DLT_RAW_BSD = 12
DLT_RAW = 101
DLT_LOOP = 108
DLT_LINUX_SLL = 113
DLT_LINUX_SLL2 = 276
class LibpcapUnavailable(RuntimeError):
pass
class _SockAddr(ctypes.Structure):
_fields_ = [("sa_family", ctypes.c_ushort), ("sa_data", ctypes.c_ubyte * 14)]
class _PcapAddr(ctypes.Structure):
pass
_PcapAddrPtr = ctypes.POINTER(_PcapAddr)
_PcapAddr._fields_ = [
("next", _PcapAddrPtr),
("addr", ctypes.POINTER(_SockAddr)),
("netmask", ctypes.POINTER(_SockAddr)),
("broadaddr", ctypes.POINTER(_SockAddr)),
("dstaddr", ctypes.POINTER(_SockAddr)),
]
class _PcapIf(ctypes.Structure):
pass
_PcapIfPtr = ctypes.POINTER(_PcapIf)
_PcapIf._fields_ = [
("next", _PcapIfPtr),
("name", ctypes.c_char_p),
("description", ctypes.c_char_p),
("addresses", _PcapAddrPtr),
("flags", ctypes.c_uint),
]
class _Timeval(ctypes.Structure):
_fields_ = [("tv_sec", ctypes.c_long), ("tv_usec", ctypes.c_long)]
class _PcapPacketHeader(ctypes.Structure):
_fields_ = [("ts", _Timeval), ("caplen", ctypes.c_uint), ("length", ctypes.c_uint)]
class _BpfProgram(ctypes.Structure):
_fields_ = [("bf_len", ctypes.c_uint), ("bf_insns", ctypes.c_void_p)]
class _PcapStat(ctypes.Structure):
# Npcap adds ps_capt after the three standard libpcap counters. Keeping the
# extra field is harmless on platforms that only populate the first three.
_fields_ = [
("ps_recv", ctypes.c_uint),
("ps_drop", ctypes.c_uint),
("ps_ifdrop", ctypes.c_uint),
("ps_capt", ctypes.c_uint),
]
@dataclass
class CaptureStats:
received: int
dropped: int
interface_dropped: int
def _windows_npcap_directory() -> Path:
return Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32" / "Npcap"
def _load_library() -> ctypes.CDLL:
candidates: list[str] = []
dll_directory = None
if sys.platform == "win32":
npcap_dir = _windows_npcap_directory()
if npcap_dir.is_dir() and hasattr(os, "add_dll_directory"):
dll_directory = os.add_dll_directory(str(npcap_dir))
candidates.extend([str(npcap_dir / "wpcap.dll"), "wpcap.dll"])
elif sys.platform == "darwin":
candidates.extend(
[
ctypes.util.find_library("pcap") or "",
"/usr/lib/libpcap.A.dylib",
"/usr/lib/libpcap.dylib",
]
)
else:
candidates.extend([ctypes.util.find_library("pcap") or "", "libpcap.so.1", "libpcap.so"])
errors = []
try:
for candidate in candidates:
if not candidate:
continue
try:
return ctypes.CDLL(candidate)
except OSError as exc:
errors.append(str(exc))
finally:
if dll_directory is not None:
dll_directory.close()
detail = f": {errors[-1]}" if errors else ""
if sys.platform == "win32":
raise LibpcapUnavailable(f"Npcap is not installed or could not be loaded{detail}")
raise LibpcapUnavailable(f"libpcap is not installed or could not be loaded{detail}")
def _configure_api(lib: ctypes.CDLL) -> None:
lib.pcap_findalldevs.argtypes = [ctypes.POINTER(_PcapIfPtr), ctypes.c_char_p]
lib.pcap_findalldevs.restype = ctypes.c_int
lib.pcap_freealldevs.argtypes = [_PcapIfPtr]
lib.pcap_create.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
lib.pcap_create.restype = ctypes.c_void_p
lib.pcap_set_snaplen.argtypes = [ctypes.c_void_p, ctypes.c_int]
lib.pcap_set_promisc.argtypes = [ctypes.c_void_p, ctypes.c_int]
lib.pcap_set_timeout.argtypes = [ctypes.c_void_p, ctypes.c_int]
lib.pcap_set_buffer_size.argtypes = [ctypes.c_void_p, ctypes.c_int]
lib.pcap_activate.argtypes = [ctypes.c_void_p]
lib.pcap_activate.restype = ctypes.c_int
lib.pcap_close.argtypes = [ctypes.c_void_p]
lib.pcap_geterr.argtypes = [ctypes.c_void_p]
lib.pcap_geterr.restype = ctypes.c_char_p
lib.pcap_datalink.argtypes = [ctypes.c_void_p]
lib.pcap_datalink.restype = ctypes.c_int
lib.pcap_compile.argtypes = [
ctypes.c_void_p,
ctypes.POINTER(_BpfProgram),
ctypes.c_char_p,
ctypes.c_int,
ctypes.c_uint,
]
lib.pcap_setfilter.argtypes = [ctypes.c_void_p, ctypes.POINTER(_BpfProgram)]
lib.pcap_freecode.argtypes = [ctypes.POINTER(_BpfProgram)]
lib.pcap_next_ex.argtypes = [
ctypes.c_void_p,
ctypes.POINTER(ctypes.POINTER(_PcapPacketHeader)),
ctypes.POINTER(ctypes.POINTER(ctypes.c_ubyte)),
]
lib.pcap_next_ex.restype = ctypes.c_int
lib.pcap_stats.argtypes = [ctypes.c_void_p, ctypes.POINTER(_PcapStat)]
lib.pcap_stats.restype = ctypes.c_int
def _pcap_error(lib: ctypes.CDLL, handle: ctypes.c_void_p) -> str:
raw = lib.pcap_geterr(handle)
return raw.decode("utf-8", errors="replace") if raw else "unknown libpcap error"
def _ipv4_from_sockaddr(address: ctypes.POINTER(_SockAddr)) -> str | None:
if not address:
return None
raw = ctypes.string_at(address, 16)
family = raw[1] if sys.platform == "darwin" else struct.unpack_from("=H", raw, 0)[0]
return socket.inet_ntoa(raw[4:8]) if family == socket.AF_INET else None
def _find_windows_device(local_ip: str) -> str | None:
if sys.platform != "win32":
return None
from ctypes import wintypes
class IpAddressString(ctypes.Structure):
_fields_ = [("value", ctypes.c_char * 16)]
class IpMaskString(ctypes.Structure):
_fields_ = [("value", ctypes.c_char * 16)]
class IpAddrString(ctypes.Structure):
pass
IpAddrStringPtr = ctypes.POINTER(IpAddrString)
IpAddrString._fields_ = [
("next", IpAddrStringPtr),
("ip_address", IpAddressString),
("ip_mask", IpMaskString),
("context", wintypes.DWORD),
]
class IpAdapterInfo(ctypes.Structure):
pass
IpAdapterInfoPtr = ctypes.POINTER(IpAdapterInfo)
IpAdapterInfo._fields_ = [
("next", IpAdapterInfoPtr),
("combo_index", wintypes.DWORD),
("adapter_name", ctypes.c_char * 260),
("description", ctypes.c_char * 132),
("address_length", wintypes.UINT),
("address", ctypes.c_ubyte * 8),
("index", wintypes.DWORD),
("adapter_type", wintypes.UINT),
("dhcp_enabled", wintypes.UINT),
("current_ip_address", IpAddrStringPtr),
("ip_address_list", IpAddrString),
("gateway_list", IpAddrString),
("dhcp_server", IpAddrString),
("have_wins", wintypes.BOOL),
("primary_wins_server", IpAddrString),
("secondary_wins_server", IpAddrString),
("lease_obtained", ctypes.c_longlong),
("lease_expires", ctypes.c_longlong),
]
ip_helper = ctypes.WinDLL("iphlpapi")
ip_helper.GetAdaptersInfo.argtypes = [IpAdapterInfoPtr, ctypes.POINTER(wintypes.ULONG)]
ip_helper.GetAdaptersInfo.restype = wintypes.DWORD
size = wintypes.ULONG(0)
if ip_helper.GetAdaptersInfo(None, ctypes.byref(size)) not in (0, 111):
return None
buffer = ctypes.create_string_buffer(size.value)
adapter = ctypes.cast(buffer, IpAdapterInfoPtr)
if ip_helper.GetAdaptersInfo(adapter, ctypes.byref(size)) != 0:
return None
while adapter:
info = adapter.contents
address = ctypes.pointer(info.ip_address_list)
while address:
ip = address.contents.ip_address.value.decode("ascii", errors="ignore")
if ip == local_ip:
adapter_name = info.adapter_name.decode("ascii", errors="ignore")
return rf"\Device\NPF_{adapter_name}"
address = address.contents.next
adapter = info.next
return None
def _find_device(lib: ctypes.CDLL, local_ip: str) -> str:
windows_device = _find_windows_device(local_ip)
if windows_device:
return windows_device
devices = _PcapIfPtr()
errbuf = ctypes.create_string_buffer(PCAP_ERRBUF_SIZE)
if lib.pcap_findalldevs(ctypes.byref(devices), errbuf) != 0:
raise LibpcapUnavailable(errbuf.value.decode("utf-8", errors="replace"))
try:
device = devices
available = []
while device:
entry = device.contents
name = entry.name.decode("utf-8", errors="replace")
addresses = entry.addresses
while addresses:
ip = _ipv4_from_sockaddr(addresses.contents.addr)
if ip:
available.append((name, ip))
if ip == local_ip:
return name
addresses = addresses.contents.next
device = entry.next
finally:
lib.pcap_freealldevs(devices)
details = ", ".join(f"{name}={ip}" for name, ip in available) or "no IPv4 capture devices"
raise LibpcapUnavailable(f"no libpcap device owns local address {local_ip}; found {details}")
def _extract_ipv4_frame(frame: bytes, datalink: int) -> bytes | None:
if datalink == DLT_EN10MB:
if len(frame) < 14:
return None
offset = 14
ether_type = struct.unpack_from("!H", frame, 12)[0]
while ether_type in (0x8100, 0x88A8, 0x9100):
if len(frame) < offset + 4:
return None
ether_type = struct.unpack_from("!H", frame, offset + 2)[0]
offset += 4
return frame[offset:] if ether_type == 0x0800 else None
if datalink in (DLT_RAW_BSD, DLT_RAW):
return frame
if datalink in (DLT_NULL, DLT_LOOP):
if len(frame) < 4:
return None
family_native = struct.unpack_from("=I", frame, 0)[0]
family_network = struct.unpack_from("!I", frame, 0)[0]
return frame[4:] if socket.AF_INET in (family_native, family_network) else None
if datalink == DLT_LINUX_SLL:
if len(frame) < 16 or struct.unpack_from("!H", frame, 14)[0] != 0x0800:
return None
return frame[16:]
if datalink == DLT_LINUX_SLL2:
if len(frame) < 20 or struct.unpack_from("!H", frame, 0)[0] != 0x0800:
return None
return frame[20:]
raise LibpcapUnavailable(f"unsupported libpcap link-layer type {datalink}")
class LibpcapCapture:
def __init__(self, local_ip: str) -> None:
self.lib = _load_library()
_configure_api(self.lib)
self.device = _find_device(self.lib, local_ip)
self.handle = ctypes.c_void_p()
errbuf = ctypes.create_string_buffer(PCAP_ERRBUF_SIZE)
handle = self.lib.pcap_create(self.device.encode("utf-8"), errbuf)
if not handle:
raise LibpcapUnavailable(errbuf.value.decode("utf-8", errors="replace"))
self.handle = ctypes.c_void_p(handle)
try:
self.lib.pcap_set_snaplen(self.handle, SNAP_LENGTH)
self.lib.pcap_set_promisc(self.handle, 0)
self.lib.pcap_set_timeout(self.handle, READ_TIMEOUT_MS)
self.lib.pcap_set_buffer_size(self.handle, CAPTURE_BUFFER_SIZE)
activation = self.lib.pcap_activate(self.handle)
if activation < 0:
raise LibpcapUnavailable(_pcap_error(self.lib, self.handle))
program = _BpfProgram()
filter_expression = f"udp and host {local_ip}".encode("ascii")
if self.lib.pcap_compile(self.handle, ctypes.byref(program), filter_expression, 1, 0xFFFFFFFF) != 0:
raise LibpcapUnavailable(_pcap_error(self.lib, self.handle))
try:
if self.lib.pcap_setfilter(self.handle, ctypes.byref(program)) != 0:
raise LibpcapUnavailable(_pcap_error(self.lib, self.handle))
finally:
self.lib.pcap_freecode(ctypes.byref(program))
self.datalink = self.lib.pcap_datalink(self.handle)
except Exception:
self.close()
raise
def packets(self) -> Iterator[ParsedIpUdpPacket | None]:
header = ctypes.POINTER(_PcapPacketHeader)()
packet_data = ctypes.POINTER(ctypes.c_ubyte)()
while self.handle:
result = self.lib.pcap_next_ex(self.handle, ctypes.byref(header), ctypes.byref(packet_data))
if result == 0:
yield None
continue
if result == -2:
return
if result < 0:
raise RuntimeError(_pcap_error(self.lib, self.handle))
frame = ctypes.string_at(packet_data, header.contents.caplen)
ipv4_packet = _extract_ipv4_frame(frame, self.datalink)
if ipv4_packet is None:
continue
packet = parse_ipv4_udp_packet(ipv4_packet)
if packet is not None:
yield packet
def stats(self) -> CaptureStats | None:
if not self.handle:
return None
stats = _PcapStat()
if self.lib.pcap_stats(self.handle, ctypes.byref(stats)) != 0:
return None
return CaptureStats(stats.ps_recv, stats.ps_drop, stats.ps_ifdrop)
def close(self) -> None:
if self.handle:
self.lib.pcap_close(self.handle)
self.handle = ctypes.c_void_p()
def open_libpcap_capture(local_ip: str) -> LibpcapCapture:
return LibpcapCapture(local_ip)

View file

@ -1,8 +1,8 @@
from __future__ import annotations
import json
import msvcrt
import socket
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path
@ -12,8 +12,10 @@ from nte_history_exporter.constants import POOL_META
from nte_history_exporter.decoder.boundary import annotate_groups, select_continuous_run_from_page_1
from nte_history_exporter.export.csv_export import write_csv
from nte_history_exporter.export.json_export import build_export_json
from nte_history_exporter.live_capture.backends import open_capture_backend
from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket
from nte_history_exporter.live_capture.windows_raw import detect_local_ipv4, open_raw_udp_socket, read_packets
from nte_history_exporter.live_capture.stop_key import StopKeyMonitor
from nte_history_exporter.live_capture.windows_raw import detect_local_ipv4
EXPORT_PREFIXES = {
@ -23,7 +25,7 @@ EXPORT_PREFIXES = {
}
def copy_to_clipboard(text: str) -> None:
def copy_to_clipboard(text: str) -> bool:
try:
import tkinter as tk
@ -33,54 +35,99 @@ def copy_to_clipboard(text: str) -> None:
root.clipboard_append(text)
root.update()
root.destroy()
return
return True
except Exception:
pass
import subprocess
commands = []
if sys.platform == "win32":
commands.append(["clip"])
elif sys.platform == "darwin":
commands.append(["pbcopy"])
else:
commands.extend([["wl-copy"], ["xclip", "-selection", "clipboard"]])
subprocess.run(["clip"], input=text, text=True, check=True)
for command in commands:
try:
subprocess.run(command, input=text, text=True, check=True)
return True
except (FileNotFoundError, subprocess.CalledProcessError):
continue
return False
def run_live_capture(
*,
interface_ip: str | None = None,
capture_backend: str = "auto",
copy_clipboard: bool = True,
write_debug_csv: bool = False,
) -> dict:
local_ip = interface_ip or detect_local_ipv4()
session = LiveHistorySession(local_ip)
sock = open_raw_udp_socket(local_ip)
console.print_live_instructions(local_ip)
capture = open_capture_backend(local_ip, capture_backend)
console.print_live_instructions(local_ip, capture.name, capture.detail)
if capture.fallback_reason:
console.print_capture_fallback(capture.fallback_reason)
reported_missing_pages: dict[str, tuple[int, ...]] = {}
try:
for packet in read_packets(sock):
if msvcrt.kbhit():
msvcrt.getch()
break
if packet is None:
continue
matched = session.process_packet(
UdpPacket(
timestamp=time.time(),
src_ip=packet.src_ip,
dst_ip=packet.dst_ip,
src_port=packet.src_port,
dst_port=packet.dst_port,
payload=packet.payload,
with StopKeyMonitor() as stop_key:
for packet in capture.packets():
if stop_key.pressed():
break
if packet is None:
continue
pair_count_before = len(session.pairs)
matched = session.process_packet(
UdpPacket(
timestamp=time.time(),
src_ip=packet.src_ip,
dst_ip=packet.dst_ip,
src_port=packet.src_port,
dst_port=packet.dst_port,
payload=packet.payload,
)
)
)
if matched:
kind = session.pairs[-1][7] if session.pairs else "permanent"
label = POOL_META.get(kind, POOL_META["permanent"])["name"]
console.print_page_captured(label, session.last_page_seen)
if matched:
affected_kinds = []
for pair in session.pairs[pair_count_before:]:
page = pair[0]
kind = pair[7] if len(pair) > 7 else "permanent"
label = POOL_META.get(kind, POOL_META["permanent"])["name"]
was_replacement = any(
existing[0] == page
and (existing[7] if len(existing) > 7 else "permanent") == kind
for existing in session.pairs[:pair_count_before]
)
console.print_page_captured(label, page, recaptured=was_replacement)
if kind not in affected_kinds:
affected_kinds.append(kind)
for kind in affected_kinds:
label = POOL_META.get(kind, POOL_META["permanent"])["name"]
missing_pages = tuple(session.missing_pages(kind))
previously_missing = reported_missing_pages.get(kind, ())
if missing_pages and missing_pages != previously_missing:
reasons = {
page: session.missing_page_reason(kind, page)
for page in missing_pages
}
console.print_missing_pages(label, list(missing_pages), reasons)
elif previously_missing and not missing_pages:
console.print_page_gap_recovered(label)
reported_missing_pages[kind] = missing_pages
finally:
try:
sock.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF)
except Exception:
pass
sock.close()
stats = capture.stats()
capture.close()
if stats:
console.print_capture_stats(
stats.received,
stats.dropped,
stats.interface_dropped,
)
exports = []
for kind in session.kinds_seen():
@ -138,8 +185,10 @@ def run_live_capture(
console.print_note(f"Export written: {item['json_path']}")
if copy_clipboard and len(exports) == 1:
copy_to_clipboard(exports[0]["payload"])
console.print_success("Export copied to clipboard - paste it straight into your tracker.")
if copy_to_clipboard(exports[0]["payload"]):
console.print_success("Export copied to clipboard - paste it straight into your tracker.")
else:
console.print_note("Clipboard tool unavailable; use the JSON file shown above.")
elif len(exports) > 1:
console.print_note("Multiple banners captured; clipboard copy skipped so one export")
console.print_note("does not overwrite another.")

View file

@ -13,10 +13,10 @@ from nte_history_exporter.decoder.arc import (
)
from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
from nte_history_exporter.decoder.protocol import (
decode_response_records,
history_request_kind,
is_history_request,
request_page,
response_contains_history_marker,
)
from nte_history_exporter.decoder.run import build_rows_from_pairs
@ -42,6 +42,8 @@ class PendingRequest:
dst_ip: str
src_port: int
dst_port: int
response_candidates: int = 0
response_candidate_lengths: tuple[int, ...] = ()
class LiveHistorySession:
@ -52,6 +54,42 @@ class LiveHistorySession:
self.packet_count = 0
self.last_match_time: float | None = None
self.last_page_seen: int | None = None
self.last_capture_was_replacement = False
self.requested_pages: dict[str, set[int]] = {}
self.unanswered_pages: dict[str, dict[int, str]] = {}
def _mark_unanswered(self, request: PendingRequest) -> None:
if request.response_candidates:
lengths = ", ".join(str(length) for length in request.response_candidate_lengths)
reason = (
f"{request.response_candidates} matching inbound UDP packet(s) captured "
f"but not recognized as history response (lengths: {lengths})"
)
else:
reason = "request captured; no matching response page was captured"
self.unanswered_pages.setdefault(request.kind, {})[request.page] = reason
def _queue_request(self, request: PendingRequest) -> None:
# The game may pipeline several page requests before responses arrive,
# and one response can contain multiple five-record pages. Keep that
# queue intact; only replace an earlier duplicate request for the same
# page, such as a recovery pass after reopening the history board.
retained = deque()
for pending in self.pending:
same_stream = (
pending.src_ip == request.src_ip
and pending.dst_ip == request.dst_ip
and pending.src_port == request.src_port
and pending.dst_port == request.dst_port
and pending.kind == request.kind
)
if same_stream and (request.page == 1 or pending.page == request.page):
self._mark_unanswered(pending)
else:
retained.append(pending)
self.pending = retained
self.requested_pages.setdefault(request.kind, set()).add(request.page)
self.pending.append(request)
def process_packet(self, packet: UdpPacket) -> bool:
self.packet_count += 1
@ -69,7 +107,7 @@ class LiveHistorySession:
src_port=packet.src_port,
dst_port=packet.dst_port,
)
self.pending.append(req)
self._queue_request(req)
self.last_page_seen = req.page
return False
@ -86,47 +124,82 @@ class LiveHistorySession:
src_port=packet.src_port,
dst_port=packet.dst_port,
)
self.pending.append(req)
self._queue_request(req)
self.last_page_seen = req.page
return False
if packet.dst_ip != self.local_ip or len(packet.payload) < 100:
return False
is_monopoly_response = response_contains_history_marker(packet.payload)
is_arc_response = bool(parse_arc_response(packet.payload))
if not is_monopoly_response and not is_arc_response:
return False
for req in self.pending:
if req.kind == "arc_miracle_box" and not is_arc_response:
continue
if req.kind != "arc_miracle_box" and not is_monopoly_response:
continue
connection_candidates = [
req
for req in self.pending
if (
packet.src_ip == req.dst_ip
and packet.dst_ip == req.src_ip
and packet.src_port == req.dst_port
and packet.dst_port == req.src_port
):
self.pending.remove(req)
self.pairs.append(
(
req.page,
req.offset,
req.request_msg,
req.request_time,
self.packet_count,
packet.timestamp,
packet.payload,
req.kind,
)
)
self.last_match_time = packet.timestamp
self.last_page_seen = req.page
return True
)
]
if not connection_candidates:
return False
return False
monopoly_records = decode_response_records(packet.payload)
arc_records = parse_arc_response(packet.payload) if not monopoly_records else []
if monopoly_records:
candidates = [req for req in connection_candidates if req.kind != "arc_miracle_box"]
records = monopoly_records
elif arc_records:
candidates = [req for req in connection_candidates if req.kind == "arc_miracle_box"]
records = arc_records
else:
candidates = connection_candidates
records = []
if not records or not candidates:
for req in candidates:
req.response_candidates += 1
req.response_candidate_lengths = (
*req.response_candidate_lengths[-4:],
len(packet.payload),
)
return False
page_count = max(1, (len(records) + 4) // 5)
if len(records) < 5:
selected = [candidates[-1]]
else:
selected = candidates[:page_count]
for page_slice, req in enumerate(selected):
self.pending.remove(req)
self.unanswered_pages.setdefault(req.kind, {}).pop(req.page, None)
slice_start = page_slice * 5
slice_count = min(5, len(records) - slice_start)
if slice_count <= 0:
break
self.last_capture_was_replacement = any(
pair[0] == req.page and (pair[7] if len(pair) > 7 else "permanent") == req.kind
for pair in self.pairs
)
self.pairs.append(
(
req.page,
req.offset,
req.request_msg,
req.request_time,
self.packet_count,
packet.timestamp,
packet.payload,
req.kind,
slice_start,
slice_count,
)
)
self.last_match_time = packet.timestamp
self.last_page_seen = req.page
return bool(selected)
def kinds_seen(self) -> list[str]:
seen = []
@ -139,6 +212,30 @@ class LiveHistorySession:
def pairs_for_kind(self, kind: str) -> list[tuple]:
return [pair for pair in self.pairs if (pair[7] if len(pair) > 7 else "permanent") == kind]
def missing_pages(self, kind: str) -> list[int]:
pages = {pair[0] for pair in self.pairs_for_kind(kind)}
if not pages:
return []
return [page for page in range(1, max(pages) + 1) if page not in pages]
def missing_page_reason(self, kind: str, page: int) -> str:
unanswered = self.unanswered_pages.get(kind, {})
if page in unanswered:
return unanswered[page]
pending = next(
(request for request in self.pending if request.kind == kind and request.page == page),
None,
)
if pending and pending.response_candidates:
lengths = ", ".join(str(length) for length in pending.response_candidate_lengths)
return (
f"{pending.response_candidates} matching inbound UDP packet(s) captured "
f"but not recognized as history response (lengths: {lengths})"
)
if page in self.requested_pages.get(kind, set()):
return "request captured; no matching response page was captured"
return "request was not captured"
def build_rows(self, kind: str | None = None) -> list[dict[str, Any]]:
if kind == "arc_miracle_box":
best_run, _warnings = select_continuous_arc_run(self.pairs_for_kind(kind))

View file

@ -0,0 +1,49 @@
from __future__ import annotations
import os
import select
import sys
class StopKeyMonitor:
def __init__(self) -> None:
self._windows = os.name == "nt"
self._fd: int | None = None
self._original_terminal = None
def __enter__(self) -> StopKeyMonitor:
if self._windows or not sys.stdin.isatty():
return self
import termios
import tty
self._fd = sys.stdin.fileno()
self._original_terminal = termios.tcgetattr(self._fd)
tty.setcbreak(self._fd)
return self
def pressed(self) -> bool:
if self._windows:
import msvcrt
if not msvcrt.kbhit():
return False
msvcrt.getch()
return True
if self._fd is None:
return False
readable, _, _ = select.select([self._fd], [], [], 0)
if not readable:
return False
os.read(self._fd, 1)
return True
def __exit__(self, _exc_type, _exc, _tb) -> None:
if self._fd is None or self._original_terminal is None:
return
import termios
termios.tcsetattr(self._fd, termios.TCSADRAIN, self._original_terminal)

View file

@ -6,6 +6,8 @@ from dataclasses import dataclass
from nte_history_exporter.live_capture.session import UdpPacket
RECEIVE_BUFFER_SIZE = 4 * 1024 * 1024
@dataclass
class ParsedIpUdpPacket:
@ -63,6 +65,7 @@ def parse_ipv4_udp_packet(data: bytes) -> ParsedIpUdpPacket | None:
def open_raw_udp_socket(local_ip: str) -> socket.socket:
sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_IP)
sock.bind((local_ip, 0))
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, RECEIVE_BUFFER_SIZE)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
sock.ioctl(socket.SIO_RCVALL, socket.RCVALL_ON)
sock.settimeout(0.5)