Merge pull request #2 from Golumpa/dev/0.1.6

Dev/0.1.6

docs/export-format.md

@@ -1,6 +1,10 @@
 # Export Format
 
-The sanitized JSON export uses:
+The JSON export is the format to use when building tools around captured NTE
+history. It is cleaned for import/use and does not include raw packet bytes or
+decoder-only offsets.
+
+## JSON shape
 
 ```json
 {
@@ -32,28 +36,80 @@ The sanitized JSON export uses:
 }
 ```
 
-Record UIDs are deterministic:
+Top-level fields:
+
+- `format` / `format_version`: Identify this export format.
+- `game`: Game name.
+- `source` / `capture_source`: Where the export came from.
+- `exporter`: Exporter name and version.
+- `banner`: The history pool this file belongs to.
+- `scan`: Export counts and capture warnings.
+- `user_uid`: Optional game account UID, if known.
+- `records`: Pull/reward records.
+
+## Fields to identify pulls
+
+For most tools, prefer these fields:
+
+- `uid`: Stable unique ID for this exported pull/reward row.
+- `user_uid`: Account UID, when present. Use this with `uid` if storing data for
+  multiple accounts.
+- `pool_group_id`: Stable pool ID for the record.
+- `banner.id`: Stable pool ID for the whole file. This should match
+  `pool_group_id` on records.
+- `timestamp`: Display timestamp from the game history.
+- `timestamp_group_ordinal`: Stable ordering inside records that share the same
+  timestamp.
+- `reward_id`: Stable decoded reward ID.
+- `reward_type`: Reward category, such as `character`, `item`, or `arc`.
+- `quantity`: Reward quantity, for Monopoly records.
+- `roll_result` / `result_type`: Monopoly result details.
+
+Current stable pool IDs:
+
+- `Lottery_Permanent`: Standard Board.
+- `Lottery_LimitedCharacter`: Limited Character Board. New limited character
+  banners should still use this ID while they share the same history/pity pool.
+- `Arc_MiracleBox`: Arc Miracle Box.
+
+Avoid using `banner.name`, `reward_name`, or `reward_rank` as primary IDs. They
+are useful display fields, but may change when mapping files are updated.
+
+## Stability notes
+
+Every JSON record is exported with a stable `uid`. Re-scanning deeper history can
+add older rows, but already exported rows keep the same `uid`.
+
+Limited character banners are grouped by their shared history pool, not by the
+currently featured character. If the game adds a new visible limited character
+banner that uses the same pool, tools should continue treating it as
+`Lottery_LimitedCharacter`.
+
+If the game adds a genuinely new history pool, the exporter mappings need to be
+updated before tools can identify it cleanly. Reward display data also comes from
+the mapping files, so new rewards may export with stable IDs before they have
+nice names or ranks.
+
+## UID generation
+
+The `uid` is the first 32 hex characters of `sha256(source)`.
+
+Monopoly source:
+
+```text
+nte|monopoly|pool_group_id|timestamp_raw|timestamp_group_ordinal|roll_result|reward_key_hex|quantity
+```
+
+Arc source:
+
+```text
+nte|gashapon|pool_group_id|timestamp_raw|timestamp_group_ordinal|reward_key_hex
+```
+
+## Example records
 
 Monopoly:
 
-```text
-nte|monopoly|Lottery_Permanent|timestamp_raw|timestamp_group_ordinal|roll_result|reward_key_hex|quantity
-```
-
-Limited character records use `Lottery_LimitedCharacter` in the same UID source position.
-
-Arc:
-
-```text
-nte|gashapon|Arc_MiracleBox|timestamp_raw|timestamp_group_ordinal|reward_key_hex
-```
-
-The final UID is `sha256(source).hexdigest()[0:32]`.
-
-Each record includes `pool_group_id`. Arc records use the shared `reward_*` fields and include `source_type: "miracle_box"`.
-
-Example Monopoly pull record:
-
 ```json
 {
   "uid": "7aae34232160e950ecc7b5da38812caa",
@@ -70,7 +126,7 @@ Example Monopoly pull record:
 }
 ```
 
-Example Arc record:
+Arc:
 
 ```json
 {
@@ -86,7 +142,7 @@ Example Arc record:
 }
 ```
 
-Normal JSON exports do not include raw packets or capture-only metadata.
+## CSV diagnostics
-`user_uid` is included when detected automatically or supplied with `--user-uid`.
+
-`capture_source` records the capture backend/source used for the export, such as
+CSV exports are mainly for debugging the decoder. Tools should prefer JSON
-`npcap`, `libpcap`, `windows_packet`, or `mitmproxy_flows`.
+unless they specifically need raw packet details.

src/nte_history_exporter/export/csv_export.py

@@ -4,7 +4,10 @@ import csv
 from pathlib import Path
 from typing import Any
 
+from nte_history_exporter import __version__
+
 FIELDNAMES = [
+    "exporter_version",
     "uid",
     "uid_status",
     "export_record",
@@ -54,4 +57,4 @@ def write_csv(path: str | Path, rows: list[dict[str, Any]]) -> None:
     with Path(path).open("w", newline="", encoding="utf-8") as f:
         writer = csv.DictWriter(f, fieldnames=FIELDNAMES, extrasaction="ignore")
         writer.writeheader()
-        writer.writerows(rows)
+        writer.writerows({**row, "exporter_version": __version__} for row in rows)

tests/test_boundary_export.py

@@ -4,6 +4,7 @@ import sys
 import unittest
 from pathlib import Path
 from unittest.mock import Mock, patch
+from tempfile import TemporaryDirectory
 
 ROOT = Path(__file__).resolve().parents[1]
 SRC = ROOT / "src"
@@ -13,6 +14,7 @@ ARC_EXPORTS = EXPORTS / "arc"
 if str(SRC) not in sys.path:
     sys.path.insert(0, str(SRC))
 
+from nte_history_exporter import __version__
 from nte_history_exporter.decoder.boundary import annotate_groups, make_uid
 from nte_history_exporter.constants import LIMITED_CHARACTER_MARKER, MARKER
 from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
@@ -21,6 +23,7 @@ from nte_history_exporter.constants import POOL_META
 from nte_history_exporter.mappings import ARC_META, CHARACTERS, ITEMS, REWARDS_BY_ID
 from nte_history_exporter.decoder.protocol import decode_reward_key, infer_reward_type
 from nte_history_exporter.decoder.user_uid import extract_user_uid
+from nte_history_exporter.export.csv_export import write_csv
 from nte_history_exporter.decoder.arc import (
     arc_request_page,
     build_arc_rows_from_pairs,
@@ -393,6 +396,17 @@ class BoundaryExportTests(unittest.TestCase):
         self.assertEqual(export["capture_source"], "npcap")
         self.assertEqual(export["user_uid"], "123456789")
 
+    def test_debug_csv_includes_exporter_version(self):
+        with TemporaryDirectory() as tmp:
+            path = Path(tmp) / "debug.csv"
+            write_csv(path, [{"uid": "abc123"}])
+
+            with path.open(newline="", encoding="utf-8") as f:
+                rows = list(csv.DictReader(f))
+
+        self.assertEqual(rows[0]["exporter_version"], __version__)
+        self.assertEqual(rows[0]["uid"], "abc123")
+
     def test_export_paths_include_user_uid_banner_and_timestamp(self):
         _csv_path, json_path = export_paths("limited_character", "218216016349")