Anchor scans to page 1; export stable prefixes

Anchor page-run selection to start at page 1 (select_continuous_run_from_page_1) instead of picking the longest run, so newest pages are preserved and the newest timestamp ordinal 0 is always captured. Change boundary logic to always assign stable UIDs and export the captured prefix of an oldest timestamp group even when it may be an unfinished 10-pull, emitting informational warnings (INCOMPLETE_TIMESTAMP_GROUP_EXPORTED / INCOMPLETE_ARC_10_PULL_EXPORTED) instead of dropping rows. Apply the same policy to Arc groups and update arc group annotation and warnings accordingly. Add a new console module for improved CLI output and update cli, live_capture, adapters, and session code to use the new run selection and console helpers. Update docs, mappings, and tests to reflect the new boundary/export behavior and messaging.
This commit is contained in:
Golumpa 2026-06-11 12:22:34 +01:00
parent 0a1a19939b
commit 20b2086299
14 changed files with 402 additions and 200 deletions

View file

@ -3,9 +3,10 @@ from __future__ import annotations
import argparse
import json
from nte_history_exporter import console
from nte_history_exporter.adapters.mitmproxy_flows import decode_mitmproxy_flows
from nte_history_exporter.decoder.arc import arc_stability_warnings
from nte_history_exporter.decoder.boundary import annotate_groups, page_gap_warnings
from nte_history_exporter.decoder.boundary import annotate_groups
from nte_history_exporter.export.csv_export import write_csv
from nte_history_exporter.export.json_export import build_export_json
from nte_history_exporter.live_capture.runner import export_paths, run_live_capture
@ -21,35 +22,18 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--interface-ip", default=None, help="local IPv4 address to bind for live capture")
parser.add_argument("--no-clipboard", action="store_true", help="do not copy live exports to clipboard")
parser.add_argument("--debug", action="store_true", help="also write research CSVs next to the JSON exports")
parser.add_argument(
"--allow-boundary-records",
action="store_true",
help="Debug only: export boundary groups even when they may be partial.",
)
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
console.print_banner()
if args.live or not args.capture_source:
result = run_live_capture(
run_live_capture(
interface_ip=args.interface_ip,
copy_clipboard=not args.no_clipboard,
write_debug_csv=args.debug,
)
for item in result["exports"]:
export = item["export"]
print(
"{banner}: decoded {decoded}, exported {exported}, skipped {skipped}.".format(
banner=export["banner"]["name"],
decoded=export["scan"]["decoded_records"],
exported=export["scan"]["exported_records"],
skipped=export["scan"]["skipped_records"],
)
)
for warning in export["scan"]["warnings"]:
suffix = f" ({warning['records']} records)" if "records" in warning else ""
print(f"WARNING {warning['code']}: {warning['reason']}{suffix}")
return 0
decoded = decode_mitmproxy_flows(args.capture_source, args.flow_index)
@ -60,12 +44,8 @@ def main(argv: list[str] | None = None) -> int:
best_run = decoded["best_arc_run"]
pair_count = len(decoded["arc_pairs"])
else:
rows, warnings = annotate_groups(
decoded["rows"],
starts_from_page_1=decoded["starts_from_page_1"],
stable_only=not args.allow_boundary_records,
)
warnings = page_gap_warnings(decoded["pairs"], decoded["best_run"]) + warnings
rows, group_warnings = annotate_groups(decoded["rows"])
warnings = decoded["run_warnings"] + group_warnings
kind = decoded["best_run"][0][7] if decoded["best_run"] and len(decoded["best_run"][0]) > 7 else "permanent"
best_run = decoded["best_run"]
pair_count = len(decoded["pairs"])
@ -83,19 +63,20 @@ def main(argv: list[str] | None = None) -> int:
)
json_path.write_text(json.dumps(export, ensure_ascii=False, indent=2), encoding="utf-8")
if args.debug:
print(f"CSV written: {out_path}")
print(f"JSON written: {json_path}")
print(
"Decoded {decoded}, exported {exported}, skipped {skipped}.".format(
decoded=export["scan"]["decoded_records"],
exported=export["scan"]["exported_records"],
skipped=export["scan"]["skipped_records"],
)
console.print_results_header()
console.print_export_summary(
export["banner"]["name"],
export["scan"]["decoded_records"],
export["scan"]["exported_records"],
export["scan"]["skipped_records"],
)
for warning in warnings:
suffix = f" ({warning['records']} records)" if "records" in warning else ""
print(f"WARNING {warning['code']}: {warning['reason']}{suffix}")
console.print_warning(warning["code"], warning["reason"], warning.get("records"))
console.maybe_print_incomplete_hint(warnings)
print()
if args.debug:
console.print_note(f"CSV written: {out_path}")
console.print_note(f"Export written: {json_path}")
return 0