0.1.3 - Add user UID, capture source & TCP support
Introduce automatic user UID extraction and optional --user-uid override, include user_uid and capture_source in exported JSON, and prepend UID to export filenames (falls back to unknown). Propagate detected protocol (tcp/udp) through live/libpcap parsing and accept TCP packets for UID detection; adjust libpcap filter to capture TCP as well. Update CLI flags (--copy-clipboard semantics, --user-uid), prompt user when UID is missing, and wire UID through mitmproxy and live capture flows. Bump package/exporter version to 0.1.3 and add tests for UID extraction and filename behavior.
This commit is contained in:
Golumpa
parent
98d97cb36a
commit
df1add6cf1
15 changed files with 283 additions and 49 deletions
|
|
@ -10,7 +10,7 @@ from dataclasses import dataclass
|
|||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
from nte_history_exporter.live_capture.windows_raw import ParsedIpUdpPacket, parse_ipv4_udp_packet
|
||||
from nte_history_exporter.live_capture.windows_raw import ParsedIpUdpPacket, parse_ipv4_packet
|
||||
|
||||
PCAP_ERRBUF_SIZE = 256
|
||||
SNAP_LENGTH = 65535
|
||||
|
|
@ -351,7 +351,7 @@ class LibpcapCapture:
|
|||
raise LibpcapUnavailable(_pcap_error(self.lib, self.handle))
|
||||
|
||||
program = _BpfProgram()
|
||||
filter_expression = f"udp and host {local_ip}".encode("ascii")
|
||||
filter_expression = f"host {local_ip} and (udp or tcp)".encode("ascii")
|
||||
if self.lib.pcap_compile(self.handle, ctypes.byref(program), filter_expression, 1, 0xFFFFFFFF) != 0:
|
||||
raise LibpcapUnavailable(_pcap_error(self.lib, self.handle))
|
||||
try:
|
||||
|
|
@ -382,7 +382,7 @@ class LibpcapCapture:
|
|||
ipv4_packet = _extract_ipv4_frame(frame, self.datalink)
|
||||
if ipv4_packet is None:
|
||||
continue
|
||||
packet = parse_ipv4_udp_packet(ipv4_packet)
|
||||
packet = parse_ipv4_packet(ipv4_packet)
|
||||
if packet is not None:
|
||||
yield packet
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,10 @@ EXPORT_PREFIXES = {
|
|||
"arc_miracle_box": "Arc",
|
||||
}
|
||||
|
||||
CAPTURE_SOURCE_LABELS = {
|
||||
"windows_raw": "windows_packet",
|
||||
}
|
||||
|
||||
|
||||
def copy_to_clipboard(text: str) -> bool:
|
||||
try:
|
||||
|
|
@ -60,8 +64,9 @@ def run_live_capture(
|
|||
*,
|
||||
interface_ip: str | None = None,
|
||||
capture_backend: str = "auto",
|
||||
copy_clipboard: bool = True,
|
||||
copy_clipboard: bool = False,
|
||||
write_debug_csv: bool = False,
|
||||
user_uid: str | None = None,
|
||||
) -> dict:
|
||||
local_ip = interface_ip or detect_local_ipv4()
|
||||
session = LiveHistorySession(local_ip)
|
||||
|
|
@ -88,6 +93,7 @@ def run_live_capture(
|
|||
src_port=packet.src_port,
|
||||
dst_port=packet.dst_port,
|
||||
payload=packet.payload,
|
||||
protocol=packet.protocol,
|
||||
)
|
||||
)
|
||||
if matched:
|
||||
|
|
@ -130,6 +136,10 @@ def run_live_capture(
|
|||
)
|
||||
|
||||
exports = []
|
||||
resolved_user_uid = user_uid or session.user_uid
|
||||
if session.kinds_seen() and not resolved_user_uid:
|
||||
resolved_user_uid = console.prompt_user_uid()
|
||||
capture_source = CAPTURE_SOURCE_LABELS.get(capture.name, capture.name)
|
||||
for kind in session.kinds_seen():
|
||||
pairs = session.pairs_for_kind(kind)
|
||||
best_run, run_warnings = select_continuous_run_from_page_1(pairs)
|
||||
|
|
@ -138,13 +148,15 @@ def run_live_capture(
|
|||
rows = annotate_groups(rows)
|
||||
warnings = run_warnings
|
||||
pages_seen = [p[0] for p in best_run]
|
||||
csv_path, json_path = export_paths(kind)
|
||||
csv_path, json_path = export_paths(kind, resolved_user_uid)
|
||||
if write_debug_csv:
|
||||
write_csv(csv_path, rows)
|
||||
export = build_export_json(
|
||||
rows,
|
||||
warnings,
|
||||
source="live_capture",
|
||||
capture_source=capture_source,
|
||||
user_uid=resolved_user_uid,
|
||||
pages_seen=pages_seen,
|
||||
)
|
||||
payload = json.dumps(export, ensure_ascii=False, indent=2)
|
||||
|
|
@ -162,9 +174,9 @@ def run_live_capture(
|
|||
console.print_results_header()
|
||||
if not exports:
|
||||
console.print_problem("No history pages were captured.")
|
||||
console.print_note("This tool must already be running when you press Start on the")
|
||||
console.print_note("game's main menu. Log out to the main menu, enter the game")
|
||||
console.print_note("again, then reopen the history screen.")
|
||||
console.print_note("Make sure the capture backend is running, then reopen the")
|
||||
console.print_note("history screen and scroll from page 1. If no page messages")
|
||||
console.print_note("appear, return to the main menu and re-enter the game.")
|
||||
return {"exports": []}
|
||||
|
||||
for item in exports:
|
||||
|
|
@ -189,24 +201,33 @@ def run_live_capture(
|
|||
console.print_success("Export copied to clipboard - paste it straight into your tracker.")
|
||||
else:
|
||||
console.print_note("Clipboard tool unavailable; use the JSON file shown above.")
|
||||
elif len(exports) > 1:
|
||||
elif copy_clipboard and len(exports) > 1:
|
||||
console.print_note("Multiple banners captured; clipboard copy skipped so one export")
|
||||
console.print_note("does not overwrite another.")
|
||||
return {"exports": exports}
|
||||
|
||||
|
||||
def export_paths(kind: str) -> tuple[Path, Path]:
|
||||
def export_paths(kind: str, user_uid: str | None = None) -> tuple[Path, Path]:
|
||||
export_dir = Path("exports")
|
||||
export_dir.mkdir(parents=True, exist_ok=True)
|
||||
prefix = EXPORT_PREFIXES.get(kind, "History")
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
base = export_dir / f"{prefix}_{stamp}"
|
||||
uid_prefix = _safe_filename_part(user_uid) if user_uid else "unknown"
|
||||
base_name = f"{uid_prefix}_{prefix}_{stamp}"
|
||||
base = export_dir / base_name
|
||||
csv_path = base.with_suffix(".csv")
|
||||
json_path = base.with_suffix(".json")
|
||||
counter = 2
|
||||
while csv_path.exists() or json_path.exists():
|
||||
base = export_dir / f"{prefix}_{stamp}_{counter}"
|
||||
base = export_dir / f"{base_name}_{counter}"
|
||||
csv_path = base.with_suffix(".csv")
|
||||
json_path = base.with_suffix(".json")
|
||||
counter += 1
|
||||
return csv_path, json_path
|
||||
|
||||
|
||||
def _safe_filename_part(value: str | None) -> str:
|
||||
if not value:
|
||||
return "unknown"
|
||||
cleaned = "".join(ch for ch in value.strip() if ch.isalnum() or ch in ("-", "_"))
|
||||
return cleaned or "unknown"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from collections import Counter, deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -19,6 +19,7 @@ from nte_history_exporter.decoder.protocol import (
|
|||
request_page,
|
||||
)
|
||||
from nte_history_exporter.decoder.run import build_rows_from_pairs
|
||||
from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -29,6 +30,7 @@ class UdpPacket:
|
|||
src_port: int
|
||||
dst_port: int
|
||||
payload: bytes
|
||||
protocol: str = "udp"
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -57,6 +59,8 @@ class LiveHistorySession:
|
|||
self.last_capture_was_replacement = False
|
||||
self.requested_pages: dict[str, set[int]] = {}
|
||||
self.unanswered_pages: dict[str, dict[int, str]] = {}
|
||||
self.user_uid: str | None = None
|
||||
self.user_uid_candidates: Counter[str] = Counter()
|
||||
|
||||
def _mark_unanswered(self, request: PendingRequest) -> None:
|
||||
if request.response_candidates:
|
||||
|
|
@ -93,6 +97,12 @@ class LiveHistorySession:
|
|||
|
||||
def process_packet(self, packet: UdpPacket) -> bool:
|
||||
self.packet_count += 1
|
||||
candidates = extract_user_uid_candidates(packet.payload)
|
||||
if candidates:
|
||||
self.user_uid_candidates.update(candidates)
|
||||
self.user_uid = self.user_uid_candidates.most_common(1)[0][0]
|
||||
if packet.protocol != "udp":
|
||||
return False
|
||||
|
||||
if packet.src_ip == self.local_ip and is_history_request(packet.payload):
|
||||
offset = int.from_bytes(packet.payload[31:35], "little")
|
||||
|
|
|
|||
|
|
@ -10,12 +10,16 @@ RECEIVE_BUFFER_SIZE = 4 * 1024 * 1024
|
|||
|
||||
|
||||
@dataclass
|
||||
class ParsedIpUdpPacket:
|
||||
class ParsedIpPacket:
|
||||
src_ip: str
|
||||
dst_ip: str
|
||||
src_port: int
|
||||
dst_port: int
|
||||
payload: bytes
|
||||
protocol: str = "udp"
|
||||
|
||||
|
||||
ParsedIpUdpPacket = ParsedIpPacket
|
||||
|
||||
|
||||
def detect_local_ipv4() -> str:
|
||||
|
|
@ -42,24 +46,41 @@ def detect_local_ipv4() -> str:
|
|||
return candidates[0]
|
||||
|
||||
|
||||
def parse_ipv4_udp_packet(data: bytes) -> ParsedIpUdpPacket | None:
|
||||
def parse_ipv4_packet(data: bytes) -> ParsedIpPacket | None:
|
||||
if len(data) < 28:
|
||||
return None
|
||||
version_ihl = data[0]
|
||||
if version_ihl >> 4 != 4:
|
||||
return None
|
||||
ihl = (version_ihl & 0x0F) * 4
|
||||
total_length = struct.unpack_from("!H", data, 2)[0]
|
||||
if total_length > 0:
|
||||
data = data[:total_length]
|
||||
if len(data) < ihl + 8:
|
||||
return None
|
||||
protocol = data[9]
|
||||
if protocol != 17:
|
||||
return None
|
||||
|
||||
src_ip = socket.inet_ntoa(data[12:16])
|
||||
dst_ip = socket.inet_ntoa(data[16:20])
|
||||
src_port, dst_port, udp_len, _checksum = struct.unpack_from("!HHHH", data, ihl)
|
||||
payload = data[ihl + 8 : ihl + udp_len]
|
||||
return ParsedIpUdpPacket(src_ip, dst_ip, src_port, dst_port, payload)
|
||||
if protocol == 17:
|
||||
src_port, dst_port, udp_len, _checksum = struct.unpack_from("!HHHH", data, ihl)
|
||||
payload = data[ihl + 8 : ihl + udp_len]
|
||||
return ParsedIpPacket(src_ip, dst_ip, src_port, dst_port, payload, "udp")
|
||||
if protocol == 6:
|
||||
if len(data) < ihl + 20:
|
||||
return None
|
||||
src_port, dst_port = struct.unpack_from("!HH", data, ihl)
|
||||
tcp_header_len = (data[ihl + 12] >> 4) * 4
|
||||
if tcp_header_len < 20 or len(data) < ihl + tcp_header_len:
|
||||
return None
|
||||
payload = data[ihl + tcp_header_len :]
|
||||
return ParsedIpPacket(src_ip, dst_ip, src_port, dst_port, payload, "tcp")
|
||||
return None
|
||||
|
||||
|
||||
def parse_ipv4_udp_packet(data: bytes) -> ParsedIpPacket | None:
|
||||
packet = parse_ipv4_packet(data)
|
||||
return packet if packet and packet.protocol == "udp" else None
|
||||
|
||||
|
||||
def open_raw_udp_socket(local_ip: str) -> socket.socket:
|
||||
|
|
@ -79,7 +100,7 @@ def read_packets(sock: socket.socket):
|
|||
except socket.timeout:
|
||||
yield None
|
||||
continue
|
||||
packet = parse_ipv4_udp_packet(data)
|
||||
packet = parse_ipv4_packet(data)
|
||||
if packet is None:
|
||||
continue
|
||||
yield packet
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue