Anchor scans to page 1; export stable prefixes
Anchor page-run selection to start at page 1 (select_continuous_run_from_page_1) instead of picking the longest run, so newest pages are preserved and the newest timestamp ordinal 0 is always captured. Change boundary logic to always assign stable UIDs and export the captured prefix of an oldest timestamp group even when it may be an unfinished 10-pull, emitting informational warnings (INCOMPLETE_TIMESTAMP_GROUP_EXPORTED / INCOMPLETE_ARC_10_PULL_EXPORTED) instead of dropping rows. Apply the same policy to Arc groups and update arc group annotation and warnings accordingly. Add a new console module for improved CLI output and update cli, live_capture, adapters, and session code to use the new run selection and console helpers. Update docs, mappings, and tests to reflect the new boundary/export behavior and messaging.
This commit is contained in:
parent
0a1a19939b
commit
20b2086299
14 changed files with 402 additions and 200 deletions
|
|
@ -4,7 +4,7 @@ import struct
|
|||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from nte_history_exporter.decoder.boundary import longest_monotonic_page_run
|
||||
from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
|
||||
from nte_history_exporter.decoder.arc import (
|
||||
arc_request_page,
|
||||
build_arc_rows_from_pairs,
|
||||
|
|
@ -128,7 +128,7 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d
|
|||
if response_index is not None:
|
||||
pairs.append((page, offset, i, ts, response_index, response_ts, response_content, kind))
|
||||
|
||||
best_run = longest_monotonic_page_run(pairs)
|
||||
best_run, run_warnings = select_continuous_run_from_page_1(pairs)
|
||||
rows_out = build_rows_from_pairs(best_run)
|
||||
best_arc_run, arc_warnings = select_continuous_arc_run(arc_pairs)
|
||||
arc_rows = build_arc_rows_from_pairs(best_arc_run)
|
||||
|
|
@ -137,10 +137,10 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d
|
|||
"flow_index": resolved_flow_index,
|
||||
"pairs": pairs,
|
||||
"best_run": best_run,
|
||||
"run_warnings": run_warnings,
|
||||
"rows": rows_out,
|
||||
"arc_pairs": arc_pairs,
|
||||
"best_arc_run": best_arc_run,
|
||||
"arc_rows": arc_rows,
|
||||
"arc_warnings": arc_warnings,
|
||||
"starts_from_page_1": bool(best_run and best_run[0][0] == 1),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@ from __future__ import annotations
|
|||
import argparse
|
||||
import json
|
||||
|
||||
from nte_history_exporter import console
|
||||
from nte_history_exporter.adapters.mitmproxy_flows import decode_mitmproxy_flows
|
||||
from nte_history_exporter.decoder.arc import arc_stability_warnings
|
||||
from nte_history_exporter.decoder.boundary import annotate_groups, page_gap_warnings
|
||||
from nte_history_exporter.decoder.boundary import annotate_groups
|
||||
from nte_history_exporter.export.csv_export import write_csv
|
||||
from nte_history_exporter.export.json_export import build_export_json
|
||||
from nte_history_exporter.live_capture.runner import export_paths, run_live_capture
|
||||
|
|
@ -21,35 +22,18 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
parser.add_argument("--interface-ip", default=None, help="local IPv4 address to bind for live capture")
|
||||
parser.add_argument("--no-clipboard", action="store_true", help="do not copy live exports to clipboard")
|
||||
parser.add_argument("--debug", action="store_true", help="also write research CSVs next to the JSON exports")
|
||||
parser.add_argument(
|
||||
"--allow-boundary-records",
|
||||
action="store_true",
|
||||
help="Debug only: export boundary groups even when they may be partial.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
console.print_banner()
|
||||
if args.live or not args.capture_source:
|
||||
result = run_live_capture(
|
||||
run_live_capture(
|
||||
interface_ip=args.interface_ip,
|
||||
copy_clipboard=not args.no_clipboard,
|
||||
write_debug_csv=args.debug,
|
||||
)
|
||||
for item in result["exports"]:
|
||||
export = item["export"]
|
||||
print(
|
||||
"{banner}: decoded {decoded}, exported {exported}, skipped {skipped}.".format(
|
||||
banner=export["banner"]["name"],
|
||||
decoded=export["scan"]["decoded_records"],
|
||||
exported=export["scan"]["exported_records"],
|
||||
skipped=export["scan"]["skipped_records"],
|
||||
)
|
||||
)
|
||||
for warning in export["scan"]["warnings"]:
|
||||
suffix = f" ({warning['records']} records)" if "records" in warning else ""
|
||||
print(f"WARNING {warning['code']}: {warning['reason']}{suffix}")
|
||||
return 0
|
||||
|
||||
decoded = decode_mitmproxy_flows(args.capture_source, args.flow_index)
|
||||
|
|
@ -60,12 +44,8 @@ def main(argv: list[str] | None = None) -> int:
|
|||
best_run = decoded["best_arc_run"]
|
||||
pair_count = len(decoded["arc_pairs"])
|
||||
else:
|
||||
rows, warnings = annotate_groups(
|
||||
decoded["rows"],
|
||||
starts_from_page_1=decoded["starts_from_page_1"],
|
||||
stable_only=not args.allow_boundary_records,
|
||||
)
|
||||
warnings = page_gap_warnings(decoded["pairs"], decoded["best_run"]) + warnings
|
||||
rows, group_warnings = annotate_groups(decoded["rows"])
|
||||
warnings = decoded["run_warnings"] + group_warnings
|
||||
kind = decoded["best_run"][0][7] if decoded["best_run"] and len(decoded["best_run"][0]) > 7 else "permanent"
|
||||
best_run = decoded["best_run"]
|
||||
pair_count = len(decoded["pairs"])
|
||||
|
|
@ -83,19 +63,20 @@ def main(argv: list[str] | None = None) -> int:
|
|||
)
|
||||
json_path.write_text(json.dumps(export, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
if args.debug:
|
||||
print(f"CSV written: {out_path}")
|
||||
print(f"JSON written: {json_path}")
|
||||
print(
|
||||
"Decoded {decoded}, exported {exported}, skipped {skipped}.".format(
|
||||
decoded=export["scan"]["decoded_records"],
|
||||
exported=export["scan"]["exported_records"],
|
||||
skipped=export["scan"]["skipped_records"],
|
||||
)
|
||||
console.print_results_header()
|
||||
console.print_export_summary(
|
||||
export["banner"]["name"],
|
||||
export["scan"]["decoded_records"],
|
||||
export["scan"]["exported_records"],
|
||||
export["scan"]["skipped_records"],
|
||||
)
|
||||
for warning in warnings:
|
||||
suffix = f" ({warning['records']} records)" if "records" in warning else ""
|
||||
print(f"WARNING {warning['code']}: {warning['reason']}{suffix}")
|
||||
console.print_warning(warning["code"], warning["reason"], warning.get("records"))
|
||||
console.maybe_print_incomplete_hint(warnings)
|
||||
print()
|
||||
if args.debug:
|
||||
console.print_note(f"CSV written: {out_path}")
|
||||
console.print_note(f"Export written: {json_path}")
|
||||
return 0
|
||||
|
||||
|
||||
|
|
|
|||
155
src/nte_history_exporter/console.py
Normal file
155
src/nte_history_exporter/console.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from nte_history_exporter.constants import EXPORTER_VERSION, GAME_NAME
|
||||
|
||||
WIDTH = 58
|
||||
|
||||
RESET = "\x1b[0m"
|
||||
BOLD = "\x1b[1m"
|
||||
DIM = "\x1b[2m"
|
||||
CYAN = "\x1b[36m"
|
||||
GREEN = "\x1b[32m"
|
||||
YELLOW = "\x1b[33m"
|
||||
|
||||
_ansi: bool | None = None
|
||||
|
||||
|
||||
def _enable_ansi() -> bool:
|
||||
if not hasattr(sys.stdout, "isatty") or not sys.stdout.isatty():
|
||||
return False
|
||||
if os.name != "nt":
|
||||
return True
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
kernel32 = ctypes.windll.kernel32
|
||||
handle = kernel32.GetStdHandle(-11)
|
||||
mode = ctypes.c_uint32()
|
||||
if not kernel32.GetConsoleMode(handle, ctypes.byref(mode)):
|
||||
return False
|
||||
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
|
||||
return bool(kernel32.SetConsoleMode(handle, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def ansi_enabled() -> bool:
|
||||
global _ansi
|
||||
if _ansi is None:
|
||||
_ansi = _enable_ansi()
|
||||
return _ansi
|
||||
|
||||
|
||||
def style(text: str, *codes: str) -> str:
|
||||
if not codes or not ansi_enabled():
|
||||
return text
|
||||
return "".join(codes) + text + RESET
|
||||
|
||||
|
||||
def rule(char: str = "-") -> str:
|
||||
return style(char * WIDTH, DIM)
|
||||
|
||||
|
||||
def print_banner() -> None:
|
||||
print()
|
||||
print(rule("="))
|
||||
print(style(f" NTE History Exporter v{EXPORTER_VERSION}", BOLD, CYAN))
|
||||
print(style(f" {GAME_NAME} pull history -> tracker JSON", DIM))
|
||||
print(rule("="))
|
||||
|
||||
|
||||
def print_live_instructions(local_ip: str) -> None:
|
||||
print()
|
||||
print(style(" Listening on ", DIM) + style(local_ip, BOLD))
|
||||
print()
|
||||
print(style(" How to export your pull history", BOLD))
|
||||
print(" 1. This tool must be running BEFORE you press Start on")
|
||||
print(" the game's main menu, or the game connection cannot")
|
||||
print(" be captured. Already in game? Log out to the main")
|
||||
print(" menu and enter again.")
|
||||
print(" 2. Open a supported history screen:")
|
||||
print(style(" Monopoly > Standard Board history", CYAN))
|
||||
print(style(" Monopoly > Limited Character Board history", CYAN))
|
||||
print(style(" Gashapon > Arc Miracle Box history", CYAN))
|
||||
print(" 3. Start at page 1 and scroll down through every page")
|
||||
print(" you want exported.")
|
||||
print(" 4. Scroll one page past where you plan to stop so the")
|
||||
print(" last pull group can be confirmed as complete.")
|
||||
print(" 5. You can open several boards in the same session.")
|
||||
print()
|
||||
print(style(" Waiting for history pages... press any key here when done.", BOLD, GREEN))
|
||||
print(rule())
|
||||
|
||||
|
||||
def print_page_captured(label: str, page: int | None) -> None:
|
||||
print(style(" + ", GREEN, BOLD) + label + style(f" page {page}", DIM))
|
||||
|
||||
|
||||
def print_results_header() -> None:
|
||||
print()
|
||||
print(rule())
|
||||
print(style(" Results", BOLD))
|
||||
print(rule())
|
||||
|
||||
|
||||
def print_export_summary(name: str, decoded: int, exported: int, skipped: int) -> None:
|
||||
counts = [
|
||||
style(f"decoded {decoded}", DIM),
|
||||
style(f"exported {exported}", GREEN, BOLD),
|
||||
style(f"skipped {skipped}", YELLOW if skipped else DIM),
|
||||
]
|
||||
print(f" {name:<30}" + " ".join(counts))
|
||||
|
||||
|
||||
def print_warning(code: str, reason: str, records: int | None = None) -> None:
|
||||
suffix = f" ({records} records)" if records is not None else ""
|
||||
print(style(f" ! {code}: {reason}{suffix}", YELLOW))
|
||||
|
||||
|
||||
def print_note(text: str) -> None:
|
||||
print(style(f" {text}", DIM))
|
||||
|
||||
|
||||
def print_success(text: str) -> None:
|
||||
print(style(f" {text}", GREEN, BOLD))
|
||||
|
||||
|
||||
def print_problem(text: str) -> None:
|
||||
print(style(f" {text}", YELLOW, BOLD))
|
||||
|
||||
|
||||
def print_boxed(lines: list[str], *codes: str) -> None:
|
||||
inner = WIDTH - 4
|
||||
border = " +" + "-" * (WIDTH - 2) + "+"
|
||||
print(style(border, *codes))
|
||||
for line in lines:
|
||||
print(style(" | " + line.ljust(inner) + " |", *codes))
|
||||
print(style(border, *codes))
|
||||
|
||||
|
||||
# Warning codes for an oldest timestamp group that was exported despite possibly
|
||||
# being an unfinished 10-pull. The exported rows are stable, so the warning is
|
||||
# informational and the user can ignore it once the rest is captured.
|
||||
INCOMPLETE_EXPORT_CODES = {
|
||||
"INCOMPLETE_TIMESTAMP_GROUP_EXPORTED",
|
||||
"INCOMPLETE_ARC_10_PULL_EXPORTED",
|
||||
}
|
||||
|
||||
|
||||
def maybe_print_incomplete_hint(warnings: list[dict]) -> None:
|
||||
if not any(w.get("code") in INCOMPLETE_EXPORT_CODES for w in warnings):
|
||||
return
|
||||
print()
|
||||
print_boxed(
|
||||
[
|
||||
"The INCOMPLETE_..._EXPORTED warning above is safe",
|
||||
"to ignore if the partial 10-pull it captured is",
|
||||
"already in your tracker. Otherwise restart",
|
||||
"the capture and go to a further page",
|
||||
],
|
||||
GREEN,
|
||||
BOLD,
|
||||
)
|
||||
|
|
@ -19,7 +19,7 @@ from nte_history_exporter.constants import (
|
|||
GAME_UID_PART,
|
||||
POOL_META,
|
||||
)
|
||||
from nte_history_exporter.decoder.boundary import longest_monotonic_page_run
|
||||
from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
|
||||
from nte_history_exporter.decoder.run import fmt_packet_time
|
||||
from nte_history_exporter.mappings import ARC_META
|
||||
|
||||
|
|
@ -145,24 +145,36 @@ def annotate_arc_groups(rows: list[dict[str, Any]]) -> None:
|
|||
for index, row in enumerate(rows):
|
||||
groups[row["timestamp_raw_hex"]].append(index)
|
||||
|
||||
for group_index, (timestamp_raw, indexes) in enumerate(groups.items()):
|
||||
complete = len(indexes) % 10 == 0
|
||||
group_items = list(groups.items())
|
||||
for group_index, (timestamp_raw, indexes) in enumerate(group_items):
|
||||
at_oldest_boundary = group_index == len(group_items) - 1
|
||||
# Arc pulls are always 10-pulls, so the only group that can be short is
|
||||
# the oldest one, where the scan stopped mid-10-pull. Its captured prefix
|
||||
# is ordinal-stable (ordinal 0 is captured, unseen rows append after it),
|
||||
# so export it and flag it rather than dropping it.
|
||||
incomplete = at_oldest_boundary and len(indexes) % 10 != 0
|
||||
skip_reason = (
|
||||
"arc timestamp group is not a complete 10-pull in this capture; "
|
||||
"exported rows are stable, scroll further to capture the rest"
|
||||
if incomplete
|
||||
else ""
|
||||
)
|
||||
for ordinal, index in enumerate(indexes):
|
||||
row = rows[index]
|
||||
row["timestamp_group_index"] = group_index
|
||||
row["timestamp_group_ordinal"] = ordinal
|
||||
row["timestamp_group_size_seen"] = len(indexes)
|
||||
row["uid"] = make_arc_uid(timestamp_raw, ordinal, row["reward_key_hex"])
|
||||
row["uid_status"] = "stable" if complete else "skipped_incomplete_timestamp_group"
|
||||
row["export_record"] = complete
|
||||
row["skip_reason"] = "" if complete else "arc timestamp group is not a complete 10-pull in this capture"
|
||||
row["uid_status"] = "incomplete_stable" if incomplete else "stable"
|
||||
row["export_record"] = True
|
||||
row["skip_reason"] = skip_reason
|
||||
|
||||
|
||||
def arc_stability_warnings(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
warnings = []
|
||||
seen = set()
|
||||
for row in rows:
|
||||
if row.get("export_record") is True:
|
||||
if row.get("uid_status") != "incomplete_stable":
|
||||
continue
|
||||
timestamp_raw = row["timestamp_raw_hex"]
|
||||
if timestamp_raw in seen:
|
||||
|
|
@ -171,38 +183,18 @@ def arc_stability_warnings(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|||
group = [r for r in rows if r["timestamp_raw_hex"] == timestamp_raw]
|
||||
warnings.append(
|
||||
{
|
||||
"code": "INCOMPLETE_ARC_10_PULL_DROPPED",
|
||||
"code": "INCOMPLETE_ARC_10_PULL_EXPORTED",
|
||||
"timestamp_raw": timestamp_raw,
|
||||
"timestamp_decoded": row["timestamp_decoded"],
|
||||
"records": len(group),
|
||||
"reason": "arc timestamp group is not a complete 10-pull in this capture",
|
||||
"reason": (
|
||||
"arc timestamp group is not a complete 10-pull in this capture; "
|
||||
"exported rows are stable, scroll further to capture the rest"
|
||||
),
|
||||
}
|
||||
)
|
||||
return warnings
|
||||
|
||||
|
||||
def select_continuous_arc_run(pairs: list[tuple]) -> tuple[list[tuple], list[dict[str, Any]]]:
|
||||
warnings: list[dict[str, Any]] = []
|
||||
if not pairs:
|
||||
return [], warnings
|
||||
pairs_by_page = {pair[0]: pair for pair in pairs}
|
||||
seen_pages = sorted(pairs_by_page)
|
||||
if 1 in pairs_by_page:
|
||||
selected_pages = []
|
||||
page = 1
|
||||
while page in pairs_by_page:
|
||||
selected_pages.append(page)
|
||||
page += 1
|
||||
if len(selected_pages) < len(seen_pages):
|
||||
ignored = [page for page in seen_pages if page not in selected_pages]
|
||||
warnings.append(
|
||||
{
|
||||
"code": "PAGE_GAP_DETECTED",
|
||||
"ignored_pages": ignored,
|
||||
"reason": f"Using continuous pages 1-{selected_pages[-1]}; ignored later pages {ignored}.",
|
||||
}
|
||||
)
|
||||
return [pairs_by_page[page] for page in selected_pages], warnings
|
||||
|
||||
warnings.append({"code": "DID_NOT_START_AT_PAGE_1", "reason": "Arc history scan did not start at page 1."})
|
||||
return longest_monotonic_page_run(pairs), warnings
|
||||
return select_continuous_run_from_page_1(pairs)
|
||||
|
|
|
|||
|
|
@ -40,31 +40,50 @@ def longest_monotonic_page_run(pairs: list[tuple]) -> list[tuple]:
|
|||
return max(runs, key=len) if runs else []
|
||||
|
||||
|
||||
def page_gap_warnings(pairs: list[tuple], best_run: list[tuple]) -> list[dict[str, Any]]:
|
||||
warnings: list[dict[str, Any]] = []
|
||||
if len(pairs) < 2:
|
||||
return warnings
|
||||
def select_continuous_run_from_page_1(pairs: list[tuple]) -> tuple[list[tuple], list[dict[str, Any]]]:
|
||||
"""Pick the run of pages starting at page 1 and continuing without gaps.
|
||||
|
||||
best_run_ids = {id(pair) for pair in best_run}
|
||||
ignored_pages = [pair[0] for pair in pairs if id(pair) not in best_run_ids]
|
||||
previous_page = pairs[0][0]
|
||||
for pair in pairs[1:]:
|
||||
page = pair[0]
|
||||
if page != previous_page + 1:
|
||||
warning = {
|
||||
"code": "PAGE_GAP_DETECTED",
|
||||
"previous_page": previous_page,
|
||||
"next_page": page,
|
||||
"ignored_pages": ignored_pages,
|
||||
"reason": (
|
||||
f"Page gap detected: saw page {previous_page} then page {page}. "
|
||||
"Pages outside the longest continuous run were ignored for stable dedupe. "
|
||||
"Re-scan or scroll more slowly."
|
||||
),
|
||||
}
|
||||
warnings.append(warning)
|
||||
previous_page = page
|
||||
return warnings
|
||||
History always loads page 1 first and is scrolled downward, so the page-1
|
||||
run holds the newest, contiguous history. Anchoring here (instead of the
|
||||
longest run anywhere) keeps the newest pages even when a later packet is
|
||||
lost, and guarantees the newest timestamp group's ordinal 0 is captured so
|
||||
every exported UID is stable. Pages after the first gap are ignored with a
|
||||
warning. If page 1 itself was not captured we fall back to the longest run
|
||||
and warn that the result may be unstable.
|
||||
"""
|
||||
warnings: list[dict[str, Any]] = []
|
||||
if not pairs:
|
||||
return [], warnings
|
||||
|
||||
pairs_by_page = {pair[0]: pair for pair in pairs}
|
||||
seen_pages = sorted(pairs_by_page)
|
||||
if 1 in pairs_by_page:
|
||||
selected_pages: list[int] = []
|
||||
page = 1
|
||||
while page in pairs_by_page:
|
||||
selected_pages.append(page)
|
||||
page += 1
|
||||
if len(selected_pages) < len(seen_pages):
|
||||
ignored = [p for p in seen_pages if p not in selected_pages]
|
||||
warnings.append(
|
||||
{
|
||||
"code": "PAGE_GAP_DETECTED",
|
||||
"ignored_pages": ignored,
|
||||
"reason": (
|
||||
f"Page gap detected after page {selected_pages[-1]}; "
|
||||
f"ignored later pages {ignored}. Re-scan or scroll more slowly."
|
||||
),
|
||||
}
|
||||
)
|
||||
return [pairs_by_page[p] for p in selected_pages], warnings
|
||||
|
||||
warnings.append(
|
||||
{
|
||||
"code": "DID_NOT_START_AT_PAGE_1",
|
||||
"reason": "Page 1 was not captured; results may be unstable. Re-scan from the top.",
|
||||
}
|
||||
)
|
||||
return longest_monotonic_page_run(pairs), warnings
|
||||
|
||||
|
||||
def is_dice_record(row: dict[str, Any]) -> bool:
|
||||
|
|
@ -81,12 +100,17 @@ def is_dice_record(row: dict[str, Any]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def annotate_groups(
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
starts_from_page_1: bool = True,
|
||||
stable_only: bool = True,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
def annotate_groups(rows: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
"""Assign timestamp-group ordinals and stable UIDs to every decoded row.
|
||||
|
||||
Pages are anchored at page 1 (see select_continuous_run_from_page_1), so the
|
||||
newest group's ordinal 0 is always captured and every UID is stable. The only
|
||||
nuance is the oldest group: if the capture did not reach the true end of
|
||||
history (final page full) and the dice-only count is not a complete multiple
|
||||
of 10, it may be an unfinished 10-pull continuing onto an uncaptured page. Its
|
||||
captured prefix is still ordinal-stable, so it is exported with an
|
||||
informational warning rather than withheld.
|
||||
"""
|
||||
if not rows:
|
||||
return rows, []
|
||||
|
||||
|
|
@ -110,28 +134,21 @@ def annotate_groups(
|
|||
|
||||
warnings: list[dict[str, Any]] = []
|
||||
for group_index, group in enumerate(groups):
|
||||
at_newest_boundary = group_index == 0
|
||||
at_oldest_boundary = group_index == len(groups) - 1
|
||||
dice_records_in_group = [row for row in group if is_dice_record(row)]
|
||||
dice_record_count = len(dice_records_in_group)
|
||||
# A complete multiple of 10 dice rolls proves the pull set is finished.
|
||||
# Unseen continuation rows can only be non-dice tails that sort after the
|
||||
# seen rows, so exported ordinals (and UIDs) stay stable.
|
||||
dice_record_count = sum(1 for row in group if is_dice_record(row))
|
||||
dice_complete = dice_record_count > 0 and dice_record_count % 10 == 0
|
||||
group_status = "stable"
|
||||
skip_reason = ""
|
||||
incomplete = at_oldest_boundary and not final_page_is_partial and not dice_complete
|
||||
skip_reason = (
|
||||
"oldest timestamp group may continue onto the next uncaptured page; "
|
||||
"exported rows are stable, scroll further to capture the rest"
|
||||
if incomplete
|
||||
else ""
|
||||
)
|
||||
|
||||
if at_newest_boundary and not starts_from_page_1:
|
||||
group_status = "dropped_boundary_group"
|
||||
skip_reason = "newest timestamp group may be partial because scan did not start from page 1"
|
||||
elif at_oldest_boundary and not final_page_is_partial and not dice_complete:
|
||||
group_status = "dropped_boundary_group"
|
||||
skip_reason = "oldest timestamp group may continue onto the next uncaptured page"
|
||||
|
||||
if group_status != "stable":
|
||||
if incomplete:
|
||||
warnings.append(
|
||||
{
|
||||
"code": "PARTIAL_TIMESTAMP_GROUP_DROPPED",
|
||||
"code": "INCOMPLETE_TIMESTAMP_GROUP_EXPORTED",
|
||||
"timestamp_raw": group[0].get("timestamp_raw_hex", ""),
|
||||
"timestamp_decoded": group[0].get("timestamp_decoded", ""),
|
||||
"records": len(group),
|
||||
|
|
@ -145,19 +162,9 @@ def annotate_groups(
|
|||
row["timestamp_group_ordinal"] = ordinal
|
||||
row["timestamp_group_size_seen"] = dice_record_count
|
||||
row["timestamp_group_record_size_seen"] = len(group)
|
||||
row["timestamp_group_boundary"] = ",".join(
|
||||
name
|
||||
for name, yes in [("newest", at_newest_boundary), ("oldest", at_oldest_boundary)]
|
||||
if yes
|
||||
)
|
||||
if group_status == "stable":
|
||||
row["uid_status"] = "stable"
|
||||
row["uid"] = make_uid(row, ordinal)
|
||||
row["export_record"] = True
|
||||
row["skip_reason"] = ""
|
||||
else:
|
||||
row["uid_status"] = group_status
|
||||
row["uid"] = "" if stable_only else make_uid(row, ordinal)
|
||||
row["export_record"] = not stable_only
|
||||
row["skip_reason"] = skip_reason
|
||||
row["timestamp_group_boundary"] = "oldest" if at_oldest_boundary else ("newest" if group_index == 0 else "")
|
||||
row["uid_status"] = "incomplete_stable" if incomplete else "stable"
|
||||
row["uid"] = make_uid(row, ordinal)
|
||||
row["export_record"] = True
|
||||
row["skip_reason"] = skip_reason
|
||||
return rows, warnings
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ def build_export_json(
|
|||
pool = next((meta for meta in POOL_META.values() if meta["id"] == pool_group_id), POOL_META["permanent"])
|
||||
scan: dict[str, Any] = {
|
||||
"mode": "stable_only",
|
||||
"boundary_policy": "drop_incomplete_timestamp_groups",
|
||||
"boundary_policy": "export_ordinal_stable_groups",
|
||||
"decoded_records": len(rows),
|
||||
"exported_records": len(exported),
|
||||
"skipped_records": len(rows) - len(exported),
|
||||
|
|
|
|||
|
|
@ -7,9 +7,10 @@ import time
|
|||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from nte_history_exporter import console
|
||||
from nte_history_exporter.constants import POOL_META
|
||||
from nte_history_exporter.decoder.arc import arc_stability_warnings, select_continuous_arc_run
|
||||
from nte_history_exporter.decoder.boundary import annotate_groups, page_gap_warnings
|
||||
from nte_history_exporter.decoder.arc import arc_stability_warnings
|
||||
from nte_history_exporter.decoder.boundary import annotate_groups, select_continuous_run_from_page_1
|
||||
from nte_history_exporter.export.csv_export import write_csv
|
||||
from nte_history_exporter.export.json_export import build_export_json
|
||||
from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket
|
||||
|
|
@ -52,8 +53,7 @@ def run_live_capture(
|
|||
session = LiveHistorySession(local_ip)
|
||||
|
||||
sock = open_raw_udp_socket(local_ip)
|
||||
print(f"Device 0 ready~! Listening on {local_ip}")
|
||||
print("Open either Monopoly history board. Capture will stay open until you press any key.")
|
||||
console.print_live_instructions(local_ip)
|
||||
|
||||
try:
|
||||
for packet in read_packets(sock):
|
||||
|
|
@ -75,7 +75,7 @@ def run_live_capture(
|
|||
if matched:
|
||||
kind = session.pairs[-1][7] if session.pairs else "permanent"
|
||||
label = POOL_META.get(kind, POOL_META["permanent"])["name"]
|
||||
print(f"Captured {label} page {session.last_page_seen}")
|
||||
console.print_page_captured(label, session.last_page_seen)
|
||||
finally:
|
||||
try:
|
||||
sock.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF)
|
||||
|
|
@ -86,14 +86,13 @@ def run_live_capture(
|
|||
exports = []
|
||||
for kind in session.kinds_seen():
|
||||
pairs = session.pairs_for_kind(kind)
|
||||
best_run = session.best_run(kind)
|
||||
best_run, run_warnings = select_continuous_run_from_page_1(pairs)
|
||||
rows = session.build_rows(kind)
|
||||
if kind == "arc_miracle_box":
|
||||
_arc_run, gap_warnings = select_continuous_arc_run(pairs)
|
||||
warnings = gap_warnings + arc_stability_warnings(rows)
|
||||
warnings = run_warnings + arc_stability_warnings(rows)
|
||||
else:
|
||||
rows, warnings = annotate_groups(rows, starts_from_page_1=bool(best_run and best_run[0][0] == 1))
|
||||
warnings = page_gap_warnings(pairs, best_run) + warnings
|
||||
rows, group_warnings = annotate_groups(rows)
|
||||
warnings = run_warnings + group_warnings
|
||||
pages_seen = [p[0] for p in best_run]
|
||||
csv_path, json_path = export_paths(kind)
|
||||
if write_debug_csv:
|
||||
|
|
@ -116,21 +115,40 @@ def run_live_capture(
|
|||
}
|
||||
)
|
||||
|
||||
console.print_results_header()
|
||||
if not exports:
|
||||
print("No Monopoly history pages were captured.")
|
||||
console.print_problem("No history pages were captured.")
|
||||
console.print_note("This tool must already be running when you press Start on the")
|
||||
console.print_note("game's main menu. Log out to the main menu, enter the game")
|
||||
console.print_note("again, then reopen the history screen.")
|
||||
return {"exports": []}
|
||||
|
||||
if copy_clipboard and len(exports) == 1:
|
||||
payload = exports[0]["payload"]
|
||||
copy_to_clipboard(payload)
|
||||
print("Export copied to clipboard.")
|
||||
elif len(exports) > 1:
|
||||
print("Multiple banners captured; clipboard copy skipped.")
|
||||
all_warnings = []
|
||||
for item in exports:
|
||||
scan = item["export"]["scan"]
|
||||
console.print_export_summary(
|
||||
item["export"]["banner"]["name"],
|
||||
scan["decoded_records"],
|
||||
scan["exported_records"],
|
||||
scan["skipped_records"],
|
||||
)
|
||||
for warning in scan["warnings"]:
|
||||
console.print_warning(warning["code"], warning["reason"], warning.get("records"))
|
||||
all_warnings.extend(scan["warnings"])
|
||||
console.maybe_print_incomplete_hint(all_warnings)
|
||||
|
||||
print()
|
||||
for item in exports:
|
||||
if item["csv_path"] is not None:
|
||||
print(f"CSV written: {item['csv_path']}")
|
||||
print(f"Export written: {item['json_path']}")
|
||||
console.print_note(f"CSV written: {item['csv_path']}")
|
||||
console.print_note(f"Export written: {item['json_path']}")
|
||||
|
||||
if copy_clipboard and len(exports) == 1:
|
||||
copy_to_clipboard(exports[0]["payload"])
|
||||
console.print_success("Export copied to clipboard - paste it straight into your tracker.")
|
||||
elif len(exports) > 1:
|
||||
console.print_note("Multiple banners captured; clipboard copy skipped so one export")
|
||||
console.print_note("does not overwrite another.")
|
||||
return {"exports": exports}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from nte_history_exporter.decoder.arc import (
|
|||
parse_arc_response,
|
||||
select_continuous_arc_run,
|
||||
)
|
||||
from nte_history_exporter.decoder.boundary import longest_monotonic_page_run
|
||||
from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
|
||||
from nte_history_exporter.decoder.protocol import (
|
||||
history_request_kind,
|
||||
is_history_request,
|
||||
|
|
@ -147,7 +147,5 @@ class LiveHistorySession:
|
|||
|
||||
def best_run(self, kind: str | None = None) -> list[tuple]:
|
||||
pairs = self.pairs_for_kind(kind) if kind else self.pairs
|
||||
if kind == "arc_miracle_box":
|
||||
best_run, _warnings = select_continuous_arc_run(pairs)
|
||||
return best_run
|
||||
return longest_monotonic_page_run(pairs)
|
||||
best_run, _warnings = select_continuous_run_from_page_1(pairs)
|
||||
return best_run
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue