Merge pull request #8 from Golumpa/dev/0.2.0

v0.2.0 - Enhance test suite, add mappings updater, and improve diagnostics

.github/workflows/release.yml

@@ -6,6 +6,10 @@ on:
 permissions:
   contents: write
 
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   validate-release:
     name: Validate Release
@@ -22,12 +26,11 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           version="$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")"
-          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+          if [ "${{ github.ref }}" != "refs/heads/main" ]; then
-            tag="v${version}"
+            echo "Development release dev-v${version} will be created or updated."
-          else
+            exit 0
-            safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')"
-            tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}"
           fi
+          tag="v${version}"
 
           if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then
             echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow."
@@ -167,8 +170,7 @@ jobs:
           if [ "${{ github.ref }}" = "refs/heads/main" ]; then
             tag="v${version}"
           else
-            safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')"
+            tag="dev-v${version}"
-            tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}"
           fi
           repo_url="${{ github.server_url }}/${{ github.repository }}"
           previous_tag="$(git tag --merged HEAD --sort=-v:refname | grep -E '^v[0-9]+(\.[0-9]+)*$' | grep -vx "$tag" | head -n 1 || true)"
@@ -196,7 +198,7 @@ jobs:
                 echo "$description" | sed 's/^/    /'
               fi
               echo
-            done < <(git rev-list "$commit_range")
+            done < <(git rev-list --no-merges "$commit_range")
           } > release-notes.md
 
       - name: Download Artifacts
@@ -223,11 +225,10 @@ jobs:
           version="${{ steps.package-version.outputs.version }}"
           if [ "${{ github.ref }}" = "refs/heads/main" ]; then
             tag="v${version}"
-            prerelease_args=()
+            title="v${version} Release"
           else
-            safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')"
+            tag="dev-v${version}"
-            tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}"
+            title="v${version} Development Build"
-            prerelease_args=(--prerelease)
           fi
           files=(
             nte-history-exporter.exe
@@ -238,14 +239,30 @@ jobs:
             nte-history-exporter-v${version}-x86_64-apple-darwin.zip
           )
 
-          if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then
+          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
-            echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow."
+            if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then
-            exit 1
+              echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow."
+              exit 1
+            fi
+
+            if gh release view "$tag" >/dev/null 2>&1; then
+              echo "::error title=GitHub release already exists::${tag} already has a GitHub release. Bump project.version in pyproject.toml before running the release workflow."
+              exit 1
+            fi
+
+            gh release create "$tag" "${files[@]}" --title "$title" --notes-file release-notes.md
+            exit 0
           fi
 
+          # Development releases are intentionally mutable. Keep one prerelease
+          # per package version and move it to the commit that produced the
+          # latest successful build.
+          git tag --force "$tag" "$GITHUB_SHA"
+          git push origin "refs/tags/${tag}" --force
+
           if gh release view "$tag" >/dev/null 2>&1; then
-            echo "::error title=GitHub release already exists::${tag} already has a GitHub release. Bump project.version in pyproject.toml before running the release workflow."
+            gh release edit "$tag" --title "$title" --notes-file release-notes.md --prerelease
-            exit 1
+            gh release upload "$tag" "${files[@]}" --clobber
+          else
+            gh release create "$tag" "${files[@]}" --title "$title" --notes-file release-notes.md --prerelease
           fi
-
-          gh release create "$tag" "${files[@]}" --title "${tag} Release" --notes-file release-notes.md "${prerelease_args[@]}"

.github/workflows/update-mappings.yml

@@ -0,0 +1,136 @@
+name: Update reward mappings
+
+on:
+  workflow_dispatch:
+    inputs:
+      source_ref:
+        description: NTE_Assets branch, tag, or commit
+        required: true
+        default: main
+        type: string
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-reward-mappings
+  cancel-in-progress: false
+
+jobs:
+  update:
+    name: Rebuild mappings from NTE_Assets
+    runs-on: ubuntu-latest
+
+    env:
+      AUTOMATION_BRANCH: dev/automated-mapping-update
+      NTE_ASSETS_REF: ${{ inputs.source_ref }}
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.repository.default_branch }}
+
+      - name: Get Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install project
+        run: python -m pip install -e .
+
+      - name: Rebuild reward mappings
+        run: python tools/update_mappings.py --source-ref "$NTE_ASSETS_REF" --apply
+
+      - name: Run tests
+        run: python -m unittest discover -s tests
+
+      - name: Summarize mapping changes
+        id: changes
+        shell: bash
+        run: |
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          report = json.loads(Path("build/mapping-update/mapping-update-report.json").read_text())
+          changes = report["changes"]
+          changed = sum(changes.values()) > 0
+          with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
+              output.write(f"changed={'true' if changed else 'false'}\n")
+          with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as summary:
+              summary.write("## NTE_Assets mapping update\n\n")
+              summary.write(f"Source ref: `{report['source_ref']}`\n\n")
+              summary.write(f"- Additions: {changes['additions']}\n")
+              summary.write(f"- Updates: {changes['updates']}\n")
+              summary.write(f"- Deletions: {changes['deletions']}\n")
+          PY
+
+      - name: Upload review report
+        uses: actions/upload-artifact@v6
+        with:
+          name: mapping-update-report
+          path: build/mapping-update/mapping-update-report.json
+
+      - name: Commit and push mapping changes
+        if: steps.changes.outputs.changed == 'true'
+        shell: bash
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$AUTOMATION_BRANCH"
+          git add mappings/arcs.json mappings/characters.json mappings/items.json
+          git commit -m "chore: update reward mappings from NTE_Assets"
+          git push --force origin "HEAD:refs/heads/$AUTOMATION_BRANCH"
+
+      - name: Open or update pull request
+        if: steps.changes.outputs.changed == 'true'
+        shell: bash
+        env:
+          BASE_BRANCH: ${{ github.event.repository.default_branch }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          python - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          report = json.loads(Path("build/mapping-update/mapping-update-report.json").read_text())
+          changes = report["changes"]
+          body = (
+              "Automated authoritative reward-mapping snapshot from "
+              "[Waifus-Grace/NTE_Assets](https://github.com/Waifus-Grace/NTE_Assets).\n\n"
+              f"Source ref: `{report['source_ref']}`\n\n"
+              f"- {changes['additions']} additions\n"
+              f"- {changes['updates']} updates\n"
+              f"- {changes['deletions']} deletions\n\n"
+              "The workflow rebuilt all reward mappings, resolved English names through "
+              "`Localization/en/game.json`, and passed the complete test suite. Pool mapping "
+              "files and UID inputs were not changed.\n\n"
+              f"[Workflow run]({os.environ['RUN_URL']})\n"
+          )
+          Path("build/mapping-update/pull-request-body.md").write_text(body, encoding="utf-8")
+          PY
+
+          pr_number="$(gh pr list \
+            --head "$AUTOMATION_BRANCH" \
+            --base "$BASE_BRANCH" \
+            --state open \
+            --json number \
+            --jq '.[0].number // empty')"
+
+          if [ -n "$pr_number" ]; then
+            gh pr edit "$pr_number" \
+              --title "chore: update reward mappings from NTE_Assets" \
+              --body-file build/mapping-update/pull-request-body.md
+          else
+            gh pr create \
+              --head "$AUTOMATION_BRANCH" \
+              --base "$BASE_BRANCH" \
+              --title "chore: update reward mappings from NTE_Assets" \
+              --body-file build/mapping-update/pull-request-body.md
+          fi

README.md

@@ -19,10 +19,11 @@ Prototype CLI exporter for **Neverness to Everness** pull history — decodes yo
 | Monopoly | Standard Board          | `Lottery_Permanent`        | Per-banner |
 | Monopoly | Limited Character Board | `Lottery_LimitedCharacter` | Shared     |
 | Gashapon | Arc Miracle Box         | `Arc_MiracleBox`           | Shared     |
+| Gashapon | Mystery Box             | `Gashapon_MysteryBox`      | Per-rotation |
 
 ## What It Does
 
-The exporter decodes Permanent Board, Limited Character Board, and Arc Miracle Box history pages from captured UDP data, applies conservative timestamp-boundary handling, and writes sanitized JSON suitable for tracker import.
+The exporter decodes Permanent Board, Limited Character Board, Arc Miracle Box, and Mystery Box history pages from captured UDP data, applies conservative timestamp-boundary handling, and writes sanitized JSON suitable for tracker import.
 
 > [!NOTE]
 > The import JSON contains decoded history rows and the shareable NTE user UID when it can be detected. It does **not** export tokens, account IDs, role IDs, device IDs, server IPs, raw packets, cookies, session data, or other capture metadata.
@@ -89,16 +90,22 @@ From source:
 Or simply double-click **`run-exporter.cmd`**.
 
 > [!IMPORTANT]
-> For automatic user UID detection, launch the tool **before pressing Start on the game's main menu**. If you are already in game, history capture can still work; the tool will ask for your UID before saving if it cannot detect it automatically.
+> For automatic user UID and account-server detection, launch the tool **before pressing Start on the game's main menu**. If you are already in game, history capture can still work; the tool will ask for missing account details before saving.
 
-Once running, open any supported history board in game. The tool keeps listening until you press any key. Exports are written under `exports\` as:
+Once running, open any supported history board in game. The tool keeps listening until you press any key. After it prints and saves the results, press any key again to close the exporter. Exports are written under `exports\` as:
 
 - `<user_uid>_Permanent_<date_time>.json`
 - `<user_uid>_Limited_<date_time>.json`
 - `<user_uid>_Arc_<date_time>.json`
+- `<user_uid>_MysteryBox_<date_time>.json`
 
 If the user UID is not detected automatically, the console asks for it before saving. Leaving it blank saves as `unknown_<banner>_<date_time>.json`, but may prevent import on some trackers.
 
+The export also includes `server_id` and `account_region` when known. If the
+initial TCP server-selection response was missed, the console offers Asia,
+America, Europe, and SEA as a numbered choice. The prompt can be left blank to
+omit server information.
+
 Exports are not copied to the clipboard by default. Add `--copy-clipboard` to copy a single captured banner's JSON after saving. If multiple banners are captured in the same run, clipboard copy is skipped so one banner does not overwrite another.
 
 If a page response is missed, the exporter reports the missing page number while capture is still running. Leave the exporter open, close and reopen that history board, then scroll down again. Scrolling backward within the existing view does not request the cached pages again. The replacement capture is accepted and the tool confirms when the gap has been recovered. If reopening the board still produces no page messages, return to the main menu and re-enter the game to start a fresh connection.
@@ -144,7 +151,7 @@ Decodes a `mitmproxy .flows` capture instead of listening live — used for rese
 | Flag      | Effect                                              |
 | --------- | --------------------------------------------------- |
 | `--live`  | Capture live UDP traffic instead of reading a file. |
-| `--debug` | Also write the full research CSV next to each JSON. |
+| `--debug` | Also write the research CSV and privacy-safe capture diagnostics. |
 | `--user-uid <uid>` | Override the auto-detected NTE user UID in the JSON export. |
 | `--copy-clipboard` | Copy a single live export JSON to clipboard after saving. |
 
@@ -156,13 +163,36 @@ Advanced live-capture selection:
 --capture-backend raw       Require the Windows raw-socket backend
 ```
 
-The `--debug` CSV holds any extra information that might be needed for fixing bugs. It contains no dangerous personal account data — only the raw bytes of the captured history page.
+The `--debug` CSV holds decoded research fields, including raw captured history
+records. A separate versioned `*.diagnostics.json` sidecar provides shareable
+reason codes and counts without payloads, network addresses, ports, packet
+timestamps, or user UID values. See [Capture diagnostics](docs/capture-diagnostics.md).
 
 The exporter automatically includes the shareable NTE user UID when it appears in the capture. If a short capture does not include it, the console asks before saving; you can also pass it explicitly with `--user-uid`.
 
 > [!TIP]
 > For reliable deduplication, start from page 1 and scroll through the pages. If you only want pages 1–5, scroll through to page 6 as well just to be on the safe side.
 
+## Mapping maintenance
+
+Reward metadata can be rebuilt directly from the latest NTE_Assets tables and
+English translation files. The reward snapshot may change IDs while UID inputs
+remain untouched:
+
+Run **Update reward mappings** from the GitHub Actions tab to generate and test
+the snapshot in a reviewable pull request, or run it locally:
+
+```powershell
+python tools/update_mappings.py
+```
+
+This rebuilds reviewable reward-map candidates directly from NTE_Assets under
+`build/mapping-update/`; committed mappings are untouched unless `--apply` is
+explicitly supplied. The snapshot may include additions, updates, and removals,
+while pool mappings and UID inputs remain untouched. See
+[Reward mapping updates](docs/mapping-updates.md) for the source rules and
+review workflow.
+
 ## Privacy
 
 > [!CAUTION]
@@ -176,7 +206,7 @@ History always loads page 1 first and is scrolled downward, so the exporter anch
 
 Within a timestamp group, ordinal 0 is the newest record and unseen rows can only append after the captured ones, so **every exported UID is stable** — including a partially captured oldest 10-pull. All decoded rows are therefore exported. Re-scanning later simply adds any rows that were not yet captured, with the same UIDs for the rows already seen.
 
-For Monopoly, Points Gift and Chase Reward rows stay in the timestamp group for UID ordinal generation, but only `result_type = dice` rows count toward pull-set sizing. Arc pulls are always 10-pulls. In both systems every captured group is exported, including the oldest one even if it is a partially captured pull set, because its captured prefix is ordinal-stable.
+For Monopoly, Points Gift and Chase Reward rows stay in the timestamp group for UID ordinal generation, but only `result_type = dice` rows count toward pull-set sizing. Arc pulls are always 10-pulls. Mystery Box records are single pulls and the final page may contain fewer than five records. Every captured group is exported, including an incomplete oldest pull set, because its captured prefix is ordinal-stable.
 
 ## Adapters
 

docs/capture-diagnostics.md

@@ -0,0 +1,27 @@
+# Capture diagnostics
+
+Running the exporter with `--debug` writes a UID-free `Capture_*.diagnostics.json`
+file in the export directory. This sidecar explains what the capture
+pipeline recognized, rejected, and paired without changing the public export
+format or record order. It is also written when no history page could be
+exported.
+
+The report contains bounded events, aggregate counters, reason codes, packet
+positions, payload lengths, history kinds, page numbers, and record counts. It
+does not contain packet payloads, IP addresses, ports, packet timestamps, or a
+user UID. The report format is versioned independently as
+`nte-capture-diagnostics` version 1.
+
+Useful rejection codes include:
+
+- `RESPONSE_TOO_SHORT`: a matching inbound packet could not contain a history
+  response;
+- `NO_HISTORY_MARKER`: a matching response candidate had no known history
+  marker;
+- `HISTORY_MARKER_PARSE_FAILED`: a known marker was present but neither decoder
+  produced records;
+- `RESPONSE_KIND_MISMATCH`: decoded data did not match the pending history kind;
+- `REQUEST_REPLACED`: a recovery request superseded an unanswered request.
+
+Events are capped at 200 per session. Aggregate counters continue after that
+limit and `events_omitted` records how many event entries were left out.

docs/export-format.md

@@ -15,7 +15,7 @@ decoder-only offsets.
   "capture_source": "npcap",
   "exporter": {
     "name": "nte-history-exporter",
-    "version": "0.1.10"
+    "version": "0.2.0"
   },
   "banner": {
     "id": "Lottery_Permanent",
@@ -32,6 +32,8 @@ decoder-only offsets.
     "warnings": []
   },
   "user_uid": "optional-user-uid",
+  "server_id": "23003",
+  "account_region": "EU",
   "records": []
 }
 ```
@@ -45,6 +47,11 @@ Top-level fields:
 - `banner`: The history pool this file belongs to.
 - `scan`: Export counts and capture warnings.
 - `user_uid`: Optional game account UID, if known.
+- `server_id`: Optional numeric account-server ID, detected from the initial TCP
+  connection or selected by the user.
+- `account_region`: Region code mapped from a known production `server_id`.
+  Current values are `AS`, `NA_SA`, `EU`, and `SE`. It is omitted for an
+  unrecognized server ID rather than guessed.
 - `records`: Pull/reward records.
 
 ## Fields to identify pulls
@@ -71,6 +78,8 @@ Current stable pool IDs:
 - `Lottery_LimitedCharacter`: Limited Character Board. New limited character
   banners should still use this ID while they share the same history/pity pool.
 - `Arc_MiracleBox`: Arc Miracle Box.
+- `Gashapon_MysteryBox`: Mystery Box. Records are single pulls and history is
+  not split into event rotations by the exporter.
 
 Avoid using `banner.name`, `reward_name`, `reward_type` or `reward_rank` as primary IDs. They
 are useful display fields, but may change when mapping files are updated.
@@ -110,6 +119,9 @@ Arc source:
 nte|gashapon|pool_group_id|timestamp_raw|timestamp_group_ordinal
 ```
 
+Mystery Box uses the same Gashapon UID source. Reward quantity is preserved,
+but each record represents exactly one pull and has `result_type = single_pull`.
+
 ## Example records
 
 Monopoly:
@@ -146,6 +158,24 @@ Arc:
 }
 ```
 
+Mystery Box:
+
+```json
+{
+  "uid": "2610c6e96afd64aa4a53fb9e8cebe031",
+  "pool_group_id": "Gashapon_MysteryBox",
+  "timestamp": "2026-07-08 19:09:33",
+  "timestamp_group_ordinal": 0,
+  "result_type": "single_pull",
+  "reward_type": "item",
+  "reward_id": "vehicle039",
+  "reward_name": "Draco",
+  "reward_rank": "S",
+  "quantity": 1,
+  "source_type": "mystery_box"
+}
+```
+
 ## CSV diagnostics
 
 CSV exports are mainly for debugging the decoder. Tools should prefer JSON

docs/limitations.md

@@ -1,15 +1,22 @@
 # Limitations
 
-This is a prototype and supports `Lottery_Permanent`, `Lottery_LimitedCharacter`, and `Arc_MiracleBox`.
+This is a prototype and supports `Lottery_Permanent`, `Lottery_LimitedCharacter`, `Arc_MiracleBox`, and `Gashapon_MysteryBox`.
 
 Known limitations:
 
 - Other NTE banners are not implemented yet.
+- Mystery Box history does not expose a semantic rotation ID in each decoded row. The exporter deliberately keeps one unsplit history stream and leaves rotation grouping to consumers.
 - The game appears not to provide a unique server-side roll ID in the decoded record body.
 - UIDs are generated deterministically from decoded fields and timestamp-group order.
 - A partially captured oldest timestamp group is still exported; its captured prefix has stable UIDs, and a later deeper scan adds the rest with the same UIDs.
 - Pages are anchored to the continuous run starting at page 1. Live capture reports gaps immediately and accepts replacement pages from another pass while the exporter remains open. Scrolling backward does not request cached pages again, so close and reopen the history board before rescanning. Any gap left when capture ends causes later pages to be ignored and reported as warnings.
 - Pipelined page requests and multi-page responses are supported, including observed 10-record responses containing two consecutive Monopoly pages. Non-byte-aligned response payloads are realigned before decoding.
+- Structured Monopoly and Arc blocks are parsed in enrichment/fallback mode.
+  The existing decoder remains authoritative when structured rows do not agree
+  on record count, reward ID, and timestamp.
+- Structured snapshot/segment assembly is limited to runs where every decoded
+  row came from structured fallback. Existing primary-decoder runs are never
+  reordered. Ambiguous generations retain the last proven snapshot.
 - Live capture prefers Npcap on Windows and automatically falls back to the built-in raw-socket backend if Npcap is unavailable. Linux and macOS require the system libpcap runtime.
 - The file adapter reads mitmproxy `.flows` captures for research and testing.
 - Npcap is Windows-only and is not redistributed with this project; Linux and macOS use their system libpcap.

docs/mapping-updates.md

@@ -0,0 +1,102 @@
+# Reward mapping updates
+
+`tools/update_mappings.py` rebuilds `arcs.json`, `characters.json`, and
+`items.json` directly from the current
+[`Waifus-Grace/NTE_Assets`](https://github.com/Waifus-Grace/NTE_Assets)
+tables.
+
+English display names come from `Localization/en/game.json`. The data tables
+provide localization namespace/key references and quality metadata; their
+embedded `LocalizedString` values are deliberately ignored.
+
+The generated files are an authoritative snapshot, not an additive merge.
+When NTE_Assets removes an ID, changes its casing, or updates its metadata, the
+staged mapping reflects that change. This is intentional for a live-service
+game whose asset tables evolve over time.
+
+## Review workflow
+
+### GitHub Actions
+
+Run **Update reward mappings** from the repository's Actions tab. The optional
+`source_ref` input accepts an NTE_Assets branch, tag, or commit and defaults to
+`main`.
+
+The workflow rebuilds the mappings, runs the complete test suite, uploads the
+JSON review report, and opens a pull request from
+`dev/automated-mapping-update`. Running it again updates the same open pull
+request. If the authoritative snapshot has not changed, it reports zero changes
+and does not create a commit or pull request.
+
+The repository setting **Allow GitHub Actions to create and approve pull
+requests** must be enabled for automatic pull-request creation. The workflow
+uses only the repository-scoped `GITHUB_TOKEN` and grants it `contents: write`
+and `pull-requests: write` permissions.
+
+### Local
+
+Stage the latest candidate files without touching the committed mappings:
+
+```powershell
+python tools/update_mappings.py
+```
+
+The required JSON tables are downloaded from the `main` branch by default.
+Pin a tag, branch, or commit for reproducible review:
+
+```powershell
+python tools/update_mappings.py --source-ref e9752e5963103529f0d683fd0aed4752b7dfad78
+```
+
+To use an existing local checkout without network access:
+
+```powershell
+python tools/update_mappings.py --assets-root path\to\NTE_Assets
+```
+
+The default output is `build/mapping-update/` and contains the three candidate
+mapping files plus `mapping-update-report.json`. The report lists additions,
+updates, and deletions by file and records SHA-256 hashes for every source
+table.
+
+After reviewing the report and diff, apply the validated snapshot explicitly:
+
+```powershell
+python tools/update_mappings.py --apply
+python -m pytest -q
+```
+
+For CI, `--check` exits with status 1 whenever the generated snapshot differs
+from the committed reward mappings. It still writes the staged artifacts.
+
+## Source rules
+
+- Every Arc in `DT_ForkItemData` is emitted to `arcs.json`.
+- Every character in `DT_Character` is emitted to `characters.json`.
+- Other pull rewards are selected by `GachaIllustrate` and resolved to the
+  inventory tables for quality and localization keys. Appearance-table IDs
+  provide canonical casing for glider rewards.
+- Mystery Box rewards are selected from every row in
+  `DT_GashaponLotteryGlobal`. No event IDs are hard-coded, so future rotations
+  are included automatically. Rewards resolve through the inventory, capital,
+  appearance, and vehicle-item tables using case-insensitive IDs.
+- Fashion, avatar-frame, and business-card Mystery Box rewards are categorized
+  as cosmetics; other Mystery Box rewards, including vehicle keys and
+  currencies, are categorized as items.
+- Names are resolved strictly through `Localization/en/game.json`. A missing or
+  ambiguous key fails the update instead of falling back to a DT value.
+- Character-awakening illustration entries are not independent pull rewards
+  and are excluded.
+- Orange, purple, and blue item qualities map to `S`, `A`, and `B`.
+
+## UID compatibility boundary
+
+This updater reads and writes only the three reward mapping files. It never
+reads or writes the permanent, limited, beginner, or Arc pool mapping files.
+Banner IDs, timestamps, record ordering, format version, and all UID inputs are
+therefore unchanged by mapping synchronization.
+
+Reward IDs and display metadata may change or disappear when the authoritative
+NTE_Assets snapshot changes. Existing exported history files remain unchanged;
+new exports describe rewards using the current asset snapshot. UID stability is
+independent of reward display mappings.

docs/packet-format.md

@@ -1,6 +1,51 @@
 # Packet Format Notes
 
-This prototype supports separate Monopoly and Arc/Gashapon history decoders.
+This prototype supports separate Monopoly, Arc/Gashapon, and Mystery Box history decoders.
+
+## Decoder strategy
+
+Both history paths now recognize the structured `FMonopolyLotteryRecordData`
+and `FForkLotteryRecordData` blocks, including blocks packed at a non-byte
+alignment. Structured fields include the item/count pair, pool ID, secondary
+reward data, roll result, and standard .NET timestamp.
+
+The structured parser is deliberately compatibility-gated:
+
+- The established decoder remains the primary path.
+- Structured rows enrich primary rows only when row count, reward ID, and raw
+  timestamp agree. Enrichment supplies exact quantities, missing roll details,
+  pool diagnostics, and secondary reward diagnostics.
+- If the primary decoder returns no records but a complete structured block is
+  valid, the structured records are converted into the same internal row shape
+  as a fallback.
+- Malformed, incomplete, mismatched, or ambiguous structured data is ignored;
+  it cannot overwrite a successfully decoded primary row.
+
+Structured protocol envelopes identify a history stream, page, query side, and
+segment index. For an all-structured fallback run, the snapshot assembler:
+
+- orders segments by their protocol index while retaining row order inside
+  every segment;
+- ignores exact retransmissions;
+- starts a new generation when an existing segment index changes;
+- replaces an older snapshot only when the new generation covers at least the
+  same segment range;
+- merges a partial generation only when its suffix has one unique overlap with
+  the proven snapshot; and
+- retains the proven snapshot and records an assembly warning when a merge is
+  ambiguous.
+
+Assembly never runs on a history run containing a successfully decoded primary
+row. Such runs retain their existing packet/page order exactly. Timestamp-group
+ordinals and UIDs are calculated only after any fallback assembly, using the
+same inputs and algorithms as before.
+
+`decoder_mode`, `structured_protocol_view`, `structured_pool_id`,
+`structured_generation_index`, `structured_assembly`,
+`structured_assembly_warning_count`, `secondary_reward_id`, and
+`secondary_quantity` are research/debug CSV fields.
+They are intentionally omitted from the public JSON export, whose format stays
+at version 1.
 
 ## Monopoly
 
@@ -60,3 +105,15 @@ Pages are anchored to the continuous run starting at page 1 (history always load
 - Arc timestamps use `unix_seconds = little_endian_u64(timestamp_raw) / 20000000 - 62135596800`.
 - Arc pulls are treated as 10-pull timestamp groups. Like Monopoly, every captured group is exported, including the oldest one even if the scan stopped mid-10-pull (its captured prefix is ordinal-stable).
 - Arc rows use the same `reward_type`, `reward_id`, `reward_name`, `reward_rank`, and `reward_key_hex` fields as Monopoly rows.
+
+## Mystery Box
+
+- Mystery Box requests use a 54-byte prefix.
+- Request constant: `2060` / `0x080c` at offset 26.
+- Request kind: `2110` / `0x083e` at offset 35.
+- Request cursor: offset 31, with a step of `2`; page is `cursor / 2`.
+- Responses contain an `FGashaponLotteryRecordData` structured block.
+- Each row contains the reward ID, exact quantity, a record flag, and a standard .NET timestamp.
+- Every row is one single pull regardless of its reward quantity.
+- A full response page contains 5 rows; the final page may contain 1–4 rows.
+- History is exported under `Gashapon_MysteryBox` without attempting to infer or split rotations.

mappings/arcs.json

@@ -1,38 +1,36 @@
 {
-  "fork_dustbin": {"name": "Dangerous Game", "rank": "B"},
-  "fork_vine": {"name": "Be Happy", "rank": "B"},
-  "fork_nonos": {"name": "First Step to Success", "rank": "B"},
   "fork_appliance": {"name": "\"Real Music\"", "rank": "B"},
-  "fork_Prokaryon": {"name": "Us.", "rank": "B"},
-  "fork_PaperPlane": {"name": "Clear Skies", "rank": "A"},
-  "fork_mofeikesi": {"name": "Good Boy's Grand Adventure", "rank": "S"},
-  "fork_wuhuakuang": {"name": "The Forgotten", "rank": "A"},
-  "fork_whuakuang": {"name": "The Forgotten", "rank": "A"},
-  "fork_jingmotingyuan": {"name": "Camellia Society", "rank": "S"},
-  "fork_Kite": {"name": "Watch Your Heads!", "rank": "A"},
-  "fork_yuren": {"name": "Umbrella", "rank": "A"},
-  "fork_yaodao": {"name": "Drawn Blade", "rank": "A"},
   "fork_Arachne": {"name": "Eternal Waltz", "rank": "S"},
   "fork_BitGame": {"name": "Blow up the Crowd", "rank": "S"},
   "fork_BitterCake": {"name": "The Good, The Bad, The Bitter", "rank": "A"},
   "fork_BlackBook": {"name": "Youthful Fantasy", "rank": "S"},
-  "fork_Baozhatang": {"name": "Fluff of Fearlessness", "rank": "S"},
+  "fork_BlastCandy": {"name": "Fluff of Fearlessness", "rank": "S"},
   "fork_bopu": {"name": "Cosmos Daze, Wild Reverie", "rank": "A"},
-  "fork_Quanjitang": {"name": "Fluff of Fortitude", "rank": "S"},
+  "fork_BoxingCandy": {"name": "Fluff of Fortitude", "rank": "S"},
   "fork_Butterfly": {"name": "Reality Refuge", "rank": "S"},
   "fork_Castle": {"name": "Call of the Twisted City", "rank": "A"},
-  "fork_Qiaoqiao": {"name": "Time Bandit", "rank": "A"},
+  "fork_Crowbar": {"name": "Time Bandit", "rank": "A"},
+  "fork_Door": {"name": "The Wrong Gate", "rank": "S"},
+  "fork_dustbin": {"name": "Dangerous Game", "rank": "B"},
+  "fork_GoldWool": {"name": "What's Desired", "rank": "S"},
   "fork_jiaojuan": {"name": "Shiny Days", "rank": "A"},
+  "fork_jingmotingyuan": {"name": "Camellia Society", "rank": "S"},
+  "fork_Kite": {"name": "Watch Your Heads!", "rank": "A"},
   "fork_KnightCandy": {"name": "Fluff of Ferocity", "rank": "S"},
   "fork_koinobori": {"name": "A Time Will Come", "rank": "A"},
   "fork_lingganzhongjiezhe": {"name": "Mind Royale", "rank": "A"},
+  "fork_LunarPhase": {"name": "Blushing Mirage", "rank": "S"},
   "fork_mamen": {"name": "Contemplative Cat", "rank": "S"},
+  "fork_mofeikesi": {"name": "Good Boy's Grand Adventure", "rank": "S"},
   "fork_moon": {"name": "Stellar Veil", "rank": "S"},
   "fork_MotorCandy": {"name": "Fluff of Fleetness", "rank": "S"},
   "fork_Nakupeda": {"name": "Your Happiness is Priceless", "rank": "S"},
   "fork_NestBird": {"name": "Tears Beneath the Mask", "rank": "S"},
+  "fork_nonos": {"name": "First Step to Success", "rank": "B"},
   "fork_oulaquantao": {"name": "Oraora!", "rank": "A"},
+  "fork_PaperPlane": {"name": "Clear Skies", "rank": "A"},
   "fork_PoliceRat": {"name": "Hethereau's Keeper", "rank": "S"},
+  "fork_Prokaryon": {"name": "Us.", "rank": "B"},
   "fork_rishi": {"name": "Day Off", "rank": "S"},
   "fork_Rose": {"name": "The Last Rose", "rank": "S"},
   "fork_snowman": {"name": "The Fools' Spring", "rank": "A"},
@@ -41,9 +39,11 @@
   "fork_TigerTally": {"name": "Ready-Ready", "rank": "S"},
   "fork_Time": {"name": "Marching Beyond Time", "rank": "S"},
   "fork_tuansanlang": {"name": "The Great Thief", "rank": "A"},
+  "fork_vine": {"name": "Be Happy", "rank": "B"},
   "fork_Whale": {"name": "Song of the Whale", "rank": "S"},
   "fork_worldrain": {"name": "The Rain That Shook the World", "rank": "S"},
-  "fork_Wushoutieyu": {"name": "Raging Flames", "rank": "S"},
+  "fork_wuhuakuang": {"name": "The Forgotten", "rank": "A"},
-  "fork_GoldWool": {"name": "What's Desired", "rank": "S"},
+  "fork_wushoutieyu": {"name": "Raging Flames", "rank": "S"},
-  "fork_LunarPhase": {"name": "Blushing Mirage", "rank": "S"}
+  "fork_yaodao": {"name": "Drawn Blade", "rank": "A"},
+  "fork_yuren": {"name": "Umbrella", "rank": "A"}
 }

mappings/characters.json

@@ -19,5 +19,6 @@
   "1070": {"name": "Aurelia", "rank": "A"},
   "1071": {"name": "Chaos", "rank": "S"},
   "1073": {"name": "Chiz", "rank": "S"},
+  "1075": {"name": "Iroi", "rank": "S"},
   "1076": {"name": "Shinku", "rank": "S"}
 }

mappings/items.json

@@ -1,26 +1,40 @@
 {
-  "DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"},
   "Dice_ticket_01": {"type": "item", "name": "Warp Piece", "rank": "S"},
   "Dice_ticket_02": {"type": "item", "name": "Lost Piece", "rank": "A"},
   "Dicelimite": {"type": "item", "name": "Solid Dice", "rank": "S"},
-  
+  "DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"},
-  "Fashion_Glide_1010": {"type": "cosmetic", "name": "Glider - Underboss-of-the-Underboss", "rank": "A"},
+  "Fashion_character_1004_01": {"type": "cosmetic", "name": "Gilded Rhapsody", "rank": "S"},
-  "Fashion_vehicle_1010_V008": {"type": "cosmetic", "name": "Mod Parts - Tiger Incoming! - Livery", "rank": "S"},
+  "Fashion_character_1010": {"type": "cosmetic", "name": "Phoenix Kick", "rank": "S"},
-  "Fashion_character_1010": {"type": "cosmetic", "name": "Character Skin - Phoenix Kick", "rank": "S"},
+  "Fashion_character_1052_01": {"type": "cosmetic", "name": "Priceless Orchid", "rank": "S"},
-  
+  "Fashion_character_1071_01": {"type": "cosmetic", "name": "Clear Skies", "rank": "S"},
-  "Fashion_Glide_1052": {"type": "cosmetic", "name": "Glider - Orchid Breeze", "rank": "A"},
+  "Fashion_character_1075_01": {"type": "cosmetic", "name": "Summer Dream", "rank": "S"},
-  "Fashion_vehicle_1052_V024": {"type": "cosmetic", "name": "Mod Parts - Autumn Haze - Livery", "rank": "S"},
+  "Fashion_character_1076_01": {"type": "cosmetic", "name": "Student of Terrasea", "rank": "S"},
-  "Fashion_character_1052_01": {"type": "cosmetic", "name": "Character Skin - Priceless Orchid", "rank": "S"},
+  "Fashion_Glide_1004": {"type": "cosmetic", "name": "Tomato Duo", "rank": "A"},
-  
+  "Fashion_Glide_1010": {"type": "cosmetic", "name": "Underboss-of-the-Underboss", "rank": "A"},
-  "Fashion_Glide_1004": {"type": "cosmetic", "name": "Glider - Tomato Duo", "rank": "A"},
+  "Fashion_Glide_1052": {"type": "cosmetic", "name": "Orchid Breeze", "rank": "A"},
-  "Fashion_vehicle_1004_V021": {"type": "cosmetic", "name": "Mod Parts - Tomato Cruise - Livery", "rank": "S"},
+  "Fashion_Glide_1071": {"type": "cosmetic", "name": "Skyrider", "rank": "A"},
-  "Fashion_character_1004_01": {"type": "cosmetic", "name": "Character Skin - Gilded Rhapsody", "rank": "S"},
+  "Fashion_glide_1075": {"type": "cosmetic", "name": "Skyrider", "rank": "A"},
-  
+  "Fashion_Glide_1076": {"type": "cosmetic", "name": "Overcast Canopy", "rank": "A"},
-  "Fashion_Glide_1071": {"type": "cosmetic", "name": "Glider - Skyrider", "rank": "A"},
+  "Fashion_vehicle_1004_V021": {"type": "cosmetic", "name": "Tomato Cruise - Livery", "rank": "S"},
-  "Fashion_vehicle_1071_V010": {"type": "cosmetic", "name": "Mod Parts - Novis ST-X 950 - Livery", "rank": "S"},
+  "Fashion_vehicle_1010_V008": {"type": "cosmetic", "name": "Tiger Incoming! - Livery", "rank": "S"},
-  "Fashion_character_1071_01": {"type": "cosmetic", "name": "Character Skin - Clear Skies", "rank": "S"},
+  "Fashion_vehicle_1052_V024": {"type": "cosmetic", "name": "Autumn Haze - Livery", "rank": "S"},
-  
+  "Fashion_vehicle_1071_V010": {"type": "cosmetic", "name": "Hound Blitz - Livery", "rank": "S"},
-  "Fashion_Glide_1076": {"type": "cosmetic", "name": "Glider - Overcast Canopy", "rank": "A"},
+  "Fashion_vehicle_1075_V006": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"},
-  "Fashion_vehicle_1076_V024": {"type": "cosmetic", "name": "Mod Parts - Hidden Dragon - Livery", "rank": "S"},
+  "Fashion_vehicle_1076_V024": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"},
-  "Fashion_character_1076_01": {"type": "cosmetic", "name": "Character Skin - Student of Terrasea", "rank": "S"}
+  "Fashion_vehicleSkin_1052_V016": {"type": "cosmetic", "name": "Golden Breeze - Livery", "rank": "S"},
+  "vehicle039": {"type": "item", "name": "Draco", "rank": "S"},
+  "Fashion_1051_rpg_level3_Dye1": {"type": "cosmetic", "name": "Glory Days - Braveheart", "rank": "S"},
+  "Fashion_1051_rpg_level3_Dye2": {"type": "cosmetic", "name": "Glory Days - Starfall", "rank": "S"},
+  "Fashion_1051_rpg_level3_Base": {"type": "cosmetic", "name": "Glory Days", "rank": "S"},
+  "SpecialGift_ticket": {"type": "item", "name": "Floe Cinema Ticket", "rank": "A"},
+  "Fons": {"type": "item", "name": "Fons", "rank": "A"},
+  "CharacterUpMaterial_lv3": {"type": "item", "name": "Elite Hunter Guide", "rank": "A"},
+  "Gold": {"type": "item", "name": "Beetle Coin", "rank": "B"},
+  "CityAbility_UpMaterial": {"type": "item", "name": "Dreamless Seed", "rank": "B"},
+  "SpecialGift_letter": {"type": "item", "name": "A Handwritten Letter", "rank": "S"},
+  "Frame_MangHe_shinku": {"type": "cosmetic", "name": "Ember Trail", "rank": "S"},
+  "Fashion_character_1051_07": {"type": "cosmetic", "name": "Perfect Partners", "rank": "S"},
+  "vehicle026": {"type": "item", "name": "Porsche 918 Spyder", "rank": "S"},
+  "Frame_MangHe_bsj": {"type": "cosmetic", "name": "Heartbeat Moment", "rank": "S"},
+  "BussinessCard_MangHe_bsj": {"type": "cosmetic", "name": "Trackside Shadow", "rank": "S"}
 }

mappings/mystery_box.json

@@ -0,0 +1,34 @@
+{
+  "pool_key": "mystery_box",
+  "game": "Neverness to Everness",
+  "system": {
+    "id": "gashapon",
+    "name": "Gashapon"
+  },
+  "banner": {
+    "id": "Gashapon_MysteryBox",
+    "name": "Mystery Box",
+    "shared_pity": false
+  },
+  "request": {
+    "family": "mystery_box",
+    "length": 54,
+    "constant": 2060,
+    "constant_offset": 26,
+    "request_kind": 2110,
+    "request_kind_offset": 35,
+    "cursor_offset": 31,
+    "cursor_step": 2,
+    "page_formula": "cursor / 2"
+  },
+  "response": {
+    "records_per_full_page": 5,
+    "marker": "FGashaponLotteryRecordData",
+    "pulls_per_record": 1,
+    "final_page_may_be_partial": true
+  },
+  "notes": [
+    "Mystery Box is a finite 15-draw prize pool whose rewards are removed as they are drawn.",
+    "History is exported as one pull per record and is not split into rotations by the exporter."
+  ]
+}

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nte-history-exporter"
-version = "0.1.10"
+version = "0.2.0"
 description = "Cross-platform Neverness to Everness pull-history exporter."
 requires-python = ">=3.10"
 dependencies = []

src/nte_history_exporter/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.10"
+__version__ = "0.2.0"

src/nte_history_exporter/adapters/mitmproxy_flows.py

@@ -7,7 +7,12 @@ from typing import Any
 from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
 from nte_history_exporter.decoder.arc import build_arc_rows_from_pairs, select_continuous_arc_run
 from nte_history_exporter.decoder.run import build_rows_from_pairs
+from nte_history_exporter.decoder.mystery_box import (
+    build_mystery_box_rows_from_pairs,
+    select_continuous_mystery_box_run,
+)
 from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates
+from nte_history_exporter.decoder.server_region import extract_server_id
 from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket
 
 
@@ -80,10 +85,15 @@ def find_udp_flow(flows: list[Any], preferred_index: int | None = None) -> tuple
 def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> dict[str, Any]:
     flows = read_flows(path)
     user_uid_candidates: Counter[str] = Counter()
+    server_id_candidates: Counter[str] = Counter()
     for flow in flows:
         for msg in flow.get(b"messages", []):
             user_uid_candidates.update(extract_user_uid_candidates(msg[1]))
+            server_id = extract_server_id(msg[1])
+            if server_id:
+                server_id_candidates.update([server_id])
     user_uid = user_uid_candidates.most_common(1)[0][0] if user_uid_candidates else None
+    server_id = server_id_candidates.most_common(1)[0][0] if server_id_candidates else None
 
     resolved_flow_index, flow = find_udp_flow(flows, flow_index)
     messages = flow[b"messages"]
@@ -101,12 +111,21 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d
             packet = UdpPacket(ts, remote_ip, local_ip, remote_port, local_port, content)
         session.process_packet(packet)
 
-    pairs = [pair for pair in session.pairs if pair[7] != "arc_miracle_box"]
+    pairs = [
+        pair
+        for pair in session.pairs
+        if pair[7] not in {"arc_miracle_box", "mystery_box"}
+    ]
     arc_pairs = [pair for pair in session.pairs if pair[7] == "arc_miracle_box"]
+    mystery_box_pairs = [pair for pair in session.pairs if pair[7] == "mystery_box"]
     best_run, run_warnings = select_continuous_run_from_page_1(pairs)
     rows_out = build_rows_from_pairs(best_run)
     best_arc_run, arc_warnings = select_continuous_arc_run(arc_pairs)
     arc_rows = build_arc_rows_from_pairs(best_arc_run)
+    best_mystery_box_run, mystery_box_warnings = select_continuous_mystery_box_run(
+        mystery_box_pairs
+    )
+    mystery_box_rows = build_mystery_box_rows_from_pairs(best_mystery_box_run)
 
     return {
         "flow_index": resolved_flow_index,
@@ -118,5 +137,11 @@ def decode_mitmproxy_flows(path: str | Path, flow_index: int | None = None) -> d
         "best_arc_run": best_arc_run,
         "arc_rows": arc_rows,
         "arc_warnings": arc_warnings,
+        "mystery_box_pairs": mystery_box_pairs,
+        "best_mystery_box_run": best_mystery_box_run,
+        "mystery_box_rows": mystery_box_rows,
+        "mystery_box_warnings": mystery_box_warnings,
         "user_uid": session.user_uid or user_uid,
+        "server_id": session.server_id or server_id,
+        "capture_diagnostics": session.diagnostic_report(),
     }

src/nte_history_exporter/cli.py

@@ -10,6 +10,7 @@ from nte_history_exporter.decoder.boundary import annotate_groups
 from nte_history_exporter.export.csv_export import write_csv
 from nte_history_exporter.export.json_export import build_export_json
 from nte_history_exporter.live_capture.libpcap import LibpcapUnavailable
+from nte_history_exporter.live_capture.diagnostics import new_diagnostics_path, write_capture_diagnostics
 from nte_history_exporter.live_capture.runner import export_paths, run_live_capture
 from nte_history_exporter.update_check import check_for_update
 
@@ -32,7 +33,11 @@ def build_parser() -> argparse.ArgumentParser:
         ),
     )
     parser.add_argument("--copy-clipboard", action="store_true", help="copy a single live export to clipboard")
-    parser.add_argument("--debug", action="store_true", help="also write research CSVs next to the JSON exports")
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="also write a research CSV and privacy-safe capture diagnostics",
+    )
     parser.add_argument("--user-uid", default=None, help="override the auto-detected NTE user UID in the JSON export")
     return parser
 
@@ -52,6 +57,7 @@ def main(argv: list[str] | None = None) -> int:
                 write_debug_csv=args.debug,
                 user_uid=args.user_uid,
             )
+            console.wait_for_close()
             return 0
         except (LibpcapUnavailable, PermissionError) as exc:
             console.print_problem(str(exc))
@@ -61,7 +67,13 @@ def main(argv: list[str] | None = None) -> int:
             return 1
 
     decoded = decode_mitmproxy_flows(args.capture_source, args.flow_index)
-    if decoded["arc_rows"] and not decoded["rows"]:
+    if decoded["mystery_box_rows"] and not decoded["rows"] and not decoded["arc_rows"]:
+        rows = decoded["mystery_box_rows"]
+        warnings = decoded["mystery_box_warnings"]
+        kind = "mystery_box"
+        best_run = decoded["best_mystery_box_run"]
+        pair_count = len(decoded["mystery_box_pairs"])
+    elif decoded["arc_rows"] and not decoded["rows"]:
         rows = decoded["arc_rows"]
         warnings = decoded["arc_warnings"]
         kind = "arc_miracle_box"
@@ -77,16 +89,23 @@ def main(argv: list[str] | None = None) -> int:
     resolved_user_uid = args.user_uid or decoded.get("user_uid")
     if not resolved_user_uid:
         resolved_user_uid = console.prompt_user_uid()
+    resolved_server_id = decoded.get("server_id")
+    if not resolved_server_id:
+        resolved_server_id = console.prompt_server_id()
 
     out_path, json_path = export_paths(kind, resolved_user_uid)
+    diagnostics_path = None
     if args.debug:
         write_csv(out_path, rows)
+        diagnostics_path = new_diagnostics_path(out_path.parent)
+        write_capture_diagnostics(diagnostics_path, decoded["capture_diagnostics"])
     export = build_export_json(
         rows,
         warnings,
         source="packet_capture",
         capture_source="mitmproxy_flows",
         user_uid=resolved_user_uid,
+        server_id=resolved_server_id,
         flow_index=decoded["flow_index"],
         candidate_request_response_pairs=pair_count,
         pages_seen=[p[0] for p in best_run],
@@ -105,6 +124,7 @@ def main(argv: list[str] | None = None) -> int:
     print()
     if args.debug:
         console.print_note(f"CSV written: {out_path}")
+        console.print_note(f"Diagnostics written: {diagnostics_path}")
     console.print_note(f"Export written: {json_path}")
     return 0
 

src/nte_history_exporter/console.py

@@ -4,6 +4,8 @@ import os
 import sys
 
 from nte_history_exporter.constants import EXPORTER_VERSION, GAME_NAME
+from nte_history_exporter.decoder.server_region import SERVER_REGIONS
+from nte_history_exporter.live_capture.stop_key import wait_for_keypress
 
 WIDTH = 58
 REPOSITORY_URL = "https://github.com/Golumpa/nte-exporter"
@@ -82,6 +84,7 @@ def print_live_instructions(local_ip: str, backend: str = "windows_raw", detail:
     print(style("         Monopoly  >  Standard Board history", CYAN))
     print(style("         Monopoly  >  Limited Character Board history", CYAN))
     print(style("         Gashapon  >  Arc Miracle Box history", CYAN))
+    print(style("         Gashapon  >  Mystery Box history", CYAN))
     print("    3. Start at page 1 and scroll down through every page")
     print("       you want exported.")
     print("    4. Scroll one page past where you plan to stop so the")
@@ -178,3 +181,30 @@ def prompt_user_uid() -> str | None:
     print_note("Leaving this blank may prevent import on some trackers.")
     value = input("  User UID: ").strip()
     return value or None
+
+
+def prompt_server_id() -> str | None:
+    print()
+    print_problem("Account server was not detected in this capture.")
+    print_note("Choose the server used by this account:")
+    choices = tuple(
+        (str(index), server_id, details["name"])
+        for index, (server_id, details) in enumerate(SERVER_REGIONS.items(), start=1)
+    )
+    for key, _server_id, name in choices:
+        print(f"    {key}. {name}")
+    print_note("Leave this blank to omit server information.")
+    by_key = {key: server_id for key, server_id, _name in choices}
+    while True:
+        value = input("  Server [1-4]: ").strip()
+        if not value:
+            return None
+        if value in by_key:
+            return by_key[value]
+        print_problem("Enter 1, 2, 3, or 4; or leave it blank.")
+
+
+def wait_for_close() -> None:
+    print()
+    print_success("Press any key to close the exporter.")
+    wait_for_keypress()

src/nte_history_exporter/constants.py

@@ -10,8 +10,10 @@ LIMITED_CHARACTER_BANNER_ID = "Lottery_LimitedCharacter"
 LIMITED_CHARACTER_BANNER_NAME = "Limited Character Board"
 ARC_BANNER_ID = "Arc_MiracleBox"
 ARC_BANNER_NAME = "Arc Miracle Box"
+MYSTERY_BOX_BANNER_ID = "Gashapon_MysteryBox"
+MYSTERY_BOX_BANNER_NAME = "Mystery Box"
 EXPORTER_NAME = "nte-history-exporter"
-EXPORTER_VERSION = "0.1.10"
+EXPORTER_VERSION = "0.2.0"
 
 HISTORY_REQUEST_BANNER = 4220
 HISTORY_REQUEST_LENGTH = 45
@@ -38,6 +40,12 @@ POOL_META = {
         "system": ARC_SYSTEM_NAME,
         "shared_pity": True,
     },
+    "mystery_box": {
+        "id": MYSTERY_BOX_BANNER_ID,
+        "name": MYSTERY_BOX_BANNER_NAME,
+        "system": ARC_SYSTEM_NAME,
+        "shared_pity": False,
+    },
 }
 
 ARC_HISTORY_REQUEST_BANNER = 2060
@@ -47,6 +55,15 @@ ARC_HISTORY_PAGE_CURSOR_MULTIPLIER = 2
 ARC_RESPONSE_FIRST_RECORD_OFFSET = 0x4C
 ARC_TIMESTAMP_TICKS_PER_SECOND = 20_000_000
 
+MYSTERY_BOX_HISTORY_REQUEST_LENGTH = 54
+MYSTERY_BOX_HISTORY_REQUEST_CONSTANT_OFFSET = 26
+MYSTERY_BOX_HISTORY_REQUEST_CONSTANT = 2060
+MYSTERY_BOX_HISTORY_CURSOR_OFFSET = 31
+MYSTERY_BOX_HISTORY_PAGE_CURSOR_MULTIPLIER = 2
+MYSTERY_BOX_HISTORY_REQUEST_KIND_OFFSET = 35
+MYSTERY_BOX_HISTORY_REQUEST_KIND = 2110
+MYSTERY_BOX_MARKER = b"FGashaponLotteryRecordData"
+
 TIMESTAMP_TICKS_PER_SECOND = 40_000_000
 DOTNET_UNIX_EPOCH_SECONDS = 62_135_596_800
 

src/nte_history_exporter/decoder/arc.py

@@ -22,6 +22,12 @@ from nte_history_exporter.constants import (
 from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
 from nte_history_exporter.decoder.run import fmt_packet_time
 from nte_history_exporter.mappings import ARC_META
+from nte_history_exporter.decoder.structured_protocol import (
+    StructuredProtocolAssembler,
+    StructuredRecord,
+    parse_structured_blocks,
+    parse_structured_records,
+)
 
 
 def is_arc_history_request(content: bytes) -> bool:
@@ -61,7 +67,7 @@ def decode_arc_timestamp(raw8: bytes) -> tuple[int, float, str]:
     return ticks, unix_seconds, decoded
 
 
-def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
+def _parse_legacy_arc_response(response: bytes) -> list[dict[str, Any]]:
     pos = ARC_RESPONSE_FIRST_RECORD_OFFSET
     records: list[dict[str, Any]] = []
     while pos + 4 <= len(response):
@@ -116,7 +122,70 @@ def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
     return records
 
 
-def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
+def _arc_metadata(arc_id: str) -> dict[str, Any]:
+    direct = ARC_META.get(arc_id)
+    if direct is not None:
+        return direct
+    folded = arc_id.casefold()
+    return next((meta for item_id, meta in ARC_META.items() if item_id.casefold() == folded), {})
+
+
+def structured_arc_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]:
+    records = []
+    for structured in structured_rows:
+        meta = _arc_metadata(structured.item_id)
+        records.append(
+            {
+                "record_start": structured.record_start,
+                "record_end": structured.record_end,
+                "record_len": structured.record_end - structured.record_start,
+                "reward_key_hex": "",
+                "reward_type": "arc",
+                "reward_id": structured.item_id,
+                "reward_name": meta.get("name", "UNKNOWN"),
+                "reward_rank": meta.get("rank", ""),
+                "type_key_hex": "",
+                "source_type": "miracle_box",
+                "timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(),
+                "timestamp_ticks": structured.ticks,
+                "timestamp_unix": structured.timestamp_unix,
+                "timestamp_decoded": structured.timestamp_decoded,
+                "record_hex": structured.record_hex,
+                "decoder_mode": "structured_fallback",
+                "structured_pool_id": structured.pool_id,
+                "structured_protocol_view": structured.protocol_view,
+                "structured_generation_index": structured.generation_index,
+            }
+        )
+    return records
+
+
+def _enrich_legacy_arc_rows(
+    legacy_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord]
+) -> list[dict[str, Any]]:
+    if len(legacy_rows) != len(structured_rows):
+        return legacy_rows
+    for legacy, structured in zip(legacy_rows, structured_rows):
+        if legacy.get("reward_id", "").casefold() != structured.item_id.casefold():
+            return legacy_rows
+        if legacy.get("timestamp_ticks") not in {structured.ticks, structured.ticks * 2}:
+            return legacy_rows
+    for legacy, structured in zip(legacy_rows, structured_rows):
+        legacy["decoder_mode"] = "heuristic_enriched"
+        legacy["structured_pool_id"] = structured.pool_id
+        legacy["structured_protocol_view"] = structured.protocol_view
+    return legacy_rows
+
+
+def parse_arc_response(response: bytes) -> list[dict[str, Any]]:
+    structured_rows = parse_structured_records(response, "fork")
+    legacy_rows = _parse_legacy_arc_response(response)
+    if legacy_rows:
+        return _enrich_legacy_arc_rows(legacy_rows, structured_rows)
+    return structured_arc_rows(structured_rows)
+
+
+def _build_primary_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
     pool = POOL_META["arc_miracle_box"]
     rows: list[dict[str, Any]] = []
     for pair in pairs:
@@ -146,6 +215,48 @@ def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
     return rows
 
 
+def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
+    primary_rows = _build_primary_arc_rows_from_pairs(pairs)
+    if primary_rows and any(row.get("decoder_mode") != "structured_fallback" for row in primary_rows):
+        return primary_rows
+
+    assembler = StructuredProtocolAssembler()
+    for source_index, pair in enumerate(pairs):
+        assembler.add_blocks(parse_structured_blocks(pair[6], "fork", source_index=source_index))
+    assembled = assembler.rows("fork")
+    if not assembled:
+        return primary_rows
+
+    pool = POOL_META["arc_miracle_box"]
+    rows = []
+    for row_index, (record, structured) in enumerate(
+        zip(structured_arc_rows(assembled), assembled), start=1
+    ):
+        source_index = structured.source_index or 0
+        pair = pairs[source_index]
+        page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7]
+        rows.append(
+            {
+                **record,
+                "page": page,
+                "offset": offset,
+                "row": row_index,
+                "pool_group_id": pool["id"],
+                "pool_group_name": pool["name"],
+                "request_msg": req_i,
+                "request_time_utc": fmt_packet_time(req_ts),
+                "response_msg": resp_i,
+                "response_time_utc": fmt_packet_time(resp_ts),
+                "response_len": len(response),
+                "record_count": len(assembled),
+                "structured_assembly": "snapshot_segments",
+                "structured_assembly_warning_count": len(assembler.warnings),
+            }
+        )
+    annotate_arc_groups(rows)
+    return rows
+
+
 def make_arc_uid(timestamp_raw: str, ordinal: int) -> str:
     source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal)])
     return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32]

src/nte_history_exporter/decoder/mystery_box.py

@@ -0,0 +1,208 @@
+from __future__ import annotations
+
+import hashlib
+import struct
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Any, Iterator
+
+from nte_history_exporter.constants import (
+    ARC_SYSTEM,
+    DOTNET_UNIX_EPOCH_SECONDS,
+    GAME_UID_PART,
+    MYSTERY_BOX_BANNER_ID,
+    MYSTERY_BOX_HISTORY_CURSOR_OFFSET,
+    MYSTERY_BOX_HISTORY_PAGE_CURSOR_MULTIPLIER,
+    MYSTERY_BOX_HISTORY_REQUEST_CONSTANT,
+    MYSTERY_BOX_HISTORY_REQUEST_CONSTANT_OFFSET,
+    MYSTERY_BOX_HISTORY_REQUEST_KIND,
+    MYSTERY_BOX_HISTORY_REQUEST_KIND_OFFSET,
+    MYSTERY_BOX_HISTORY_REQUEST_LENGTH,
+    MYSTERY_BOX_MARKER,
+    POOL_META,
+)
+from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
+from nte_history_exporter.decoder.protocol import infer_reward_type
+from nte_history_exporter.decoder.run import fmt_packet_time
+from nte_history_exporter.mappings import REWARDS_BY_ID
+
+DOTNET_TICKS_PER_SECOND = 10_000_000
+MAX_RECORDS_PER_BLOCK = 100
+MAX_REWARD_ID_LENGTH = 256
+REWARDS_BY_CASEFOLDED_ID = {
+    reward_id.casefold(): reward for reward_id, reward in REWARDS_BY_ID.items()
+}
+
+
+def is_mystery_box_history_request(content: bytes) -> bool:
+    if len(content) < MYSTERY_BOX_HISTORY_REQUEST_LENGTH:
+        return False
+    return (
+        struct.unpack_from("<I", content, MYSTERY_BOX_HISTORY_REQUEST_CONSTANT_OFFSET)[0]
+        == MYSTERY_BOX_HISTORY_REQUEST_CONSTANT
+        and struct.unpack_from("<I", content, MYSTERY_BOX_HISTORY_REQUEST_KIND_OFFSET)[0]
+        == MYSTERY_BOX_HISTORY_REQUEST_KIND
+    )
+
+
+def mystery_box_request_page(content: bytes) -> int:
+    cursor = struct.unpack_from("<I", content, MYSTERY_BOX_HISTORY_CURSOR_OFFSET)[0]
+    return cursor // MYSTERY_BOX_HISTORY_PAGE_CURSOR_MULTIPLIER
+
+
+def _protocol_views(payload: bytes) -> Iterator[bytes]:
+    yield payload
+    # History payloads can begin at a non-byte-aligned transport position.
+    for bit_shift in range(1, 8):
+        shifted = bytearray()
+        for index in range(max(0, len(payload) - 8)):
+            bit_pos = (8 + index) * 8 + bit_shift
+            byte_pos, shift = divmod(bit_pos, 8)
+            if byte_pos >= len(payload):
+                break
+            value = payload[byte_pos] >> shift
+            if shift and byte_pos + 1 < len(payload):
+                value |= payload[byte_pos + 1] << (8 - shift)
+            shifted.append(value & 0xFF)
+        yield bytes(shifted)
+
+
+def _decode_timestamp(raw: bytes) -> tuple[int, float, str]:
+    ticks = struct.unpack("<Q", raw)[0]
+    unix_seconds = ticks / DOTNET_TICKS_PER_SECOND - DOTNET_UNIX_EPOCH_SECONDS
+    decoded = datetime.fromtimestamp(unix_seconds, timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
+    return ticks, unix_seconds, decoded
+
+
+def _parse_view(data: bytes) -> list[dict[str, Any]]:
+    marker_pos = data.find(MYSTERY_BOX_MARKER)
+    if marker_pos < 0:
+        return []
+    pos = marker_pos + len(MYSTERY_BOX_MARKER)
+    if pos < len(data) and data[pos] == 0:
+        pos += 1
+    if pos + 12 > len(data):
+        return []
+    _reserved, _declared_size, row_count = struct.unpack_from("<III", data, pos)
+    pos += 12
+    if row_count > MAX_RECORDS_PER_BLOCK:
+        return []
+
+    rows: list[dict[str, Any]] = []
+    try:
+        for _ in range(row_count):
+            record_start = pos
+            reward_length = struct.unpack_from("<I", data, pos)[0]
+            pos += 4
+            if not 1 <= reward_length <= MAX_REWARD_ID_LENGTH or pos + reward_length + 13 > len(data):
+                return []
+            reward_raw = data[pos : pos + reward_length]
+            pos += reward_length
+            if not reward_raw.endswith(b"\0"):
+                return []
+            reward_id = reward_raw[:-1].decode("utf-8")
+            quantity = struct.unpack_from("<I", data, pos)[0]
+            pos += 4
+            record_flag = data[pos]
+            pos += 1
+            timestamp_raw = data[pos : pos + 8]
+            pos += 8
+            ticks, unix_seconds, timestamp_decoded = _decode_timestamp(timestamp_raw)
+            reward = REWARDS_BY_ID.get(reward_id) or REWARDS_BY_CASEFOLDED_ID.get(
+                reward_id.casefold(), {}
+            )
+            rows.append(
+                {
+                    "record_start": record_start,
+                    "record_end": pos,
+                    "record_len": pos - record_start,
+                    "reward_type": reward.get("type") or infer_reward_type(reward_id),
+                    "reward_id": reward_id,
+                    "reward_name": reward.get("name", ""),
+                    "reward_rank": reward.get("rank"),
+                    "quantity": quantity,
+                    "source_type": "mystery_box",
+                    "result_type": "single_pull",
+                    "timestamp_raw_hex": timestamp_raw.hex(),
+                    "timestamp_ticks": ticks,
+                    "timestamp_unix": unix_seconds,
+                    "timestamp_decoded": timestamp_decoded,
+                    "record_flag": record_flag,
+                    "record_hex": data[record_start:pos].hex(),
+                    "decoder_mode": "structured",
+                }
+            )
+    except (OSError, OverflowError, UnicodeDecodeError, ValueError, struct.error):
+        return []
+    return rows
+
+
+def parse_mystery_box_response(payload: bytes) -> list[dict[str, Any]]:
+    for view in _protocol_views(payload):
+        rows = _parse_view(view)
+        if rows:
+            return rows
+    return []
+
+
+def make_mystery_box_uid(timestamp_raw: str, ordinal: int) -> str:
+    source = "|".join(
+        [GAME_UID_PART, ARC_SYSTEM, MYSTERY_BOX_BANNER_ID, timestamp_raw, str(ordinal)]
+    )
+    return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32]
+
+
+def annotate_mystery_box_rows(rows: list[dict[str, Any]]) -> None:
+    groups: dict[str, list[int]] = defaultdict(list)
+    for index, row in enumerate(rows):
+        groups[row["timestamp_raw_hex"]].append(index)
+    for group_index, (timestamp_raw, indexes) in enumerate(groups.items()):
+        for ordinal, index in enumerate(indexes):
+            row = rows[index]
+            row["timestamp_group_index"] = group_index
+            row["timestamp_group_ordinal"] = ordinal
+            row["timestamp_group_size_seen"] = len(indexes)
+            row["timestamp_group_record_size_seen"] = len(indexes)
+            row["timestamp_group_boundary"] = (
+                "oldest" if group_index == len(groups) - 1 else ("newest" if group_index == 0 else "")
+            )
+            row["uid"] = make_mystery_box_uid(timestamp_raw, ordinal)
+            row["uid_status"] = "stable"
+            row["export_record"] = True
+            row["skip_reason"] = ""
+
+
+def build_mystery_box_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
+    pool = POOL_META["mystery_box"]
+    rows: list[dict[str, Any]] = []
+    for pair in pairs:
+        page, offset, req_i, req_ts, resp_i, resp_ts, response = pair[:7]
+        records = parse_mystery_box_response(response)
+        if len(pair) > 9:
+            slice_start, slice_count = pair[8:10]
+            records = records[slice_start : slice_start + slice_count]
+        for row_index, record in enumerate(records, start=1):
+            rows.append(
+                {
+                    **record,
+                    "page": page,
+                    "offset": offset,
+                    "row": row_index,
+                    "pool_group_id": pool["id"],
+                    "pool_group_name": pool["name"],
+                    "request_msg": req_i,
+                    "request_time_utc": fmt_packet_time(req_ts),
+                    "response_msg": resp_i,
+                    "response_time_utc": fmt_packet_time(resp_ts),
+                    "response_len": len(response),
+                    "record_count": len(records),
+                }
+            )
+    annotate_mystery_box_rows(rows)
+    return rows
+
+
+def select_continuous_mystery_box_run(
+    pairs: list[tuple],
+) -> tuple[list[tuple], list[dict[str, Any]]]:
+    return select_continuous_run_from_page_1(pairs)

src/nte_history_exporter/decoder/protocol.py

@@ -16,6 +16,10 @@ from nte_history_exporter.constants import (
     VALID_DICE_FIELDS,
 )
 from nte_history_exporter.mappings import REWARDS_BY_ID
+from nte_history_exporter.decoder.structured_protocol import (
+    StructuredRecord,
+    parse_structured_records,
+)
 
 REWARD_ID_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_")
 WARP_PIECE_CHASE_PATTERN = bytes.fromhex(
@@ -299,7 +303,117 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str,
     return rows
 
 
+def _structured_result(raw: int | None) -> tuple[int | None, int | None, str, int | None]:
+    if raw is None:
+        return None, None, "unknown", None
+    if raw == 0:
+        return 0, 0, "points_gift", 0
+    if raw == 0xFFFFFFFF:
+        return -4, -4, "chase_reward", -4
+    return raw, raw, "dice", raw
+
+
+def _structured_rows_compatible(
+    heuristic_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord]
+) -> bool:
+    if len(heuristic_rows) != len(structured_rows):
+        return False
+    for heuristic, structured in zip(heuristic_rows, structured_rows):
+        heuristic_id = heuristic.get("reward_id") or ""
+        if heuristic_id and heuristic_id.casefold() != structured.item_id.casefold():
+            return False
+        heuristic_ticks = heuristic.get("timestamp_ticks")
+        if heuristic_ticks not in {structured.ticks, structured.ticks * 4}:
+            return False
+    return True
+
+
+def _enrich_heuristic_rows(
+    heuristic_rows: list[dict[str, Any]], structured_rows: list[StructuredRecord]
+) -> list[dict[str, Any]]:
+    if not _structured_rows_compatible(heuristic_rows, structured_rows):
+        return heuristic_rows
+    for heuristic, structured in zip(heuristic_rows, structured_rows):
+        if not heuristic.get("reward_id"):
+            heuristic["reward_id"] = structured.item_id
+            reward = _reward_metadata(structured.item_id)
+            heuristic["reward_type"] = reward.get("type") or infer_reward_type(structured.item_id)
+            heuristic["reward_name"] = reward.get("name", "")
+            heuristic["reward_rank"] = reward.get("rank")
+        heuristic["quantity"] = structured.count
+        if heuristic.get("result_type") == "unknown" or heuristic.get("dice") is None:
+            dice, dice_raw, result_type, result_source = _structured_result(structured.roll_points_raw)
+            heuristic["dice"] = dice
+            heuristic["dice_raw_u32"] = dice_raw
+            heuristic["result_type"] = result_type
+            heuristic["result_source_raw"] = result_source
+            heuristic["roll_result"] = (
+                "Points Gift"
+                if result_type == "points_gift"
+                else ("Chase Reward" if result_type == "chase_reward" else (f"Dice {dice}" if dice else ""))
+            )
+        heuristic["decoder_mode"] = "heuristic_enriched"
+        heuristic["structured_pool_id"] = structured.pool_id
+        heuristic["secondary_reward_id"] = structured.secondary_item_id
+        heuristic["secondary_quantity"] = structured.secondary_count
+        heuristic["structured_protocol_view"] = structured.protocol_view
+    return heuristic_rows
+
+
+def _reward_metadata(reward_id: str) -> dict[str, Any]:
+    direct = REWARDS_BY_ID.get(reward_id)
+    if direct is not None:
+        return direct
+    folded = reward_id.casefold()
+    return next((meta for item_id, meta in REWARDS_BY_ID.items() if item_id.casefold() == folded), {})
+
+
+def structured_monopoly_rows(structured_rows: list[StructuredRecord]) -> list[dict[str, Any]]:
+    rows = []
+    for row_index, structured in enumerate(structured_rows, start=1):
+        dice, dice_raw, result_type, result_source = _structured_result(structured.roll_points_raw)
+        reward = _reward_metadata(structured.item_id)
+        rows.append(
+            {
+                "row": row_index,
+                "record_start": structured.record_start,
+                "record_end": structured.record_end,
+                "record_len": structured.record_end - structured.record_start,
+                "dice": dice,
+                "roll_result": (
+                    "Points Gift"
+                    if result_type == "points_gift"
+                    else ("Chase Reward" if result_type == "chase_reward" else (f"Dice {dice}" if dice else ""))
+                ),
+                "result_type": result_type,
+                "result_source_raw": result_source,
+                "dice_raw_u32": dice_raw,
+                "dice_offset_in_record": None,
+                "reward_key_hex": "",
+                "reward_type": reward.get("type") or infer_reward_type(structured.item_id),
+                "reward_id": structured.item_id,
+                "reward_name": reward.get("name", ""),
+                "reward_rank": reward.get("rank"),
+                "quantity": structured.count,
+                "timestamp_raw_hex": structured.ticks.to_bytes(8, "little").hex(),
+                "timestamp_ticks": structured.ticks,
+                "timestamp_unix": f"{structured.timestamp_unix:.6f}",
+                "timestamp_decoded": structured.timestamp_decoded,
+                "record_hex": structured.record_hex,
+                "decoder_mode": "structured_fallback",
+                "structured_pool_id": structured.pool_id,
+                "secondary_reward_id": structured.secondary_item_id,
+                "secondary_quantity": structured.secondary_count,
+                "structured_protocol_view": structured.protocol_view,
+                "structured_generation_index": structured.generation_index,
+            }
+        )
+    return rows
+
+
 def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
+    structured_rows = parse_structured_records(response_content, "monopoly")
+    heuristic_rows: list[dict[str, Any]] = []
     for candidate in iter_history_response_alignments(response_content):
         if not any(marker in candidate for marker in MARKERS):
             continue
@@ -308,5 +422,8 @@ def decode_response_records(response_content: bytes) -> list[dict[str, Any]]:
         except (OSError, OverflowError, ValueError):
             continue
         if rows:
-            return rows
+            heuristic_rows = rows
-    return []
+            break
+    if heuristic_rows:
+        return _enrich_heuristic_rows(heuristic_rows, structured_rows)
+    return structured_monopoly_rows(structured_rows)

src/nte_history_exporter/decoder/run.py

@@ -4,7 +4,11 @@ from datetime import datetime, timezone
 from typing import Any
 
 from nte_history_exporter.constants import POOL_META
-from nte_history_exporter.decoder.protocol import decode_response_records
+from nte_history_exporter.decoder.protocol import decode_response_records, structured_monopoly_rows
+from nte_history_exporter.decoder.structured_protocol import (
+    StructuredProtocolAssembler,
+    parse_structured_blocks,
+)
 
 
 def fmt_packet_time(ts: float | None) -> str:
@@ -13,7 +17,7 @@ def fmt_packet_time(ts: float | None) -> str:
     return datetime.fromtimestamp(ts, timezone.utc).strftime("%H:%M:%S.%f")[:-3]
 
 
-def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
+def _build_primary_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
     rows_out: list[dict[str, Any]] = []
     for pair in pairs:
         page, offset, req_i, req_ts, resp_i, resp_ts, response_content = pair[:7]
@@ -59,3 +63,47 @@ def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
                 }
             )
     return rows_out
+
+
+def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
+    primary_rows = _build_primary_rows_from_pairs(pairs)
+    decoded_rows = [row for row in primary_rows if row.get("record_count", 0)]
+    if decoded_rows and any(row.get("decoder_mode") != "structured_fallback" for row in decoded_rows):
+        return primary_rows
+
+    assembler = StructuredProtocolAssembler()
+    for source_index, pair in enumerate(pairs):
+        assembler.add_blocks(
+            parse_structured_blocks(pair[6], "monopoly", source_index=source_index)
+        )
+    assembled = assembler.rows("monopoly")
+    if not assembled:
+        return primary_rows
+
+    rows_out: list[dict[str, Any]] = []
+    converted = structured_monopoly_rows(assembled)
+    for row_index, (record, structured) in enumerate(zip(converted, assembled), start=1):
+        source_index = structured.source_index or 0
+        pair = pairs[source_index]
+        page, offset, req_i, req_ts, resp_i, resp_ts, response_content = pair[:7]
+        kind = pair[7] if len(pair) > 7 else "permanent"
+        pool = POOL_META.get(kind, POOL_META["permanent"])
+        rows_out.append(
+            {
+                "page": page,
+                "offset": offset,
+                "pool_group_id": pool["id"],
+                "pool_group_name": pool["name"],
+                "request_msg": req_i,
+                "request_time_utc": fmt_packet_time(req_ts),
+                "response_msg": resp_i,
+                "response_time_utc": fmt_packet_time(resp_ts),
+                "response_len": len(response_content),
+                "record_count": len(assembled),
+                **record,
+                "row": row_index,
+                "structured_assembly": "snapshot_segments",
+                "structured_assembly_warning_count": len(assembler.warnings),
+            }
+        )
+    return rows_out

src/nte_history_exporter/decoder/server_region.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import ipaddress
+import struct
+
+
+# Production entries from clientRes/OB-NTE-ServerList/serverlist_hhOS.json.
+# Test and certification servers are intentionally excluded.
+SERVER_REGIONS = {
+    "23001": {"account_region": "AS", "name": "Asia"},
+    "23002": {"account_region": "NA_SA", "name": "America"},
+    "23003": {"account_region": "EU", "name": "Europe"},
+    "23004": {"account_region": "SE", "name": "SEA"},
+}
+
+
+def account_region_for_server(server_id: str | None) -> str | None:
+    if not server_id:
+        return None
+    server = SERVER_REGIONS.get(str(server_id).strip())
+    return server["account_region"] if server else None
+
+
+def extract_server_id(payload: bytes) -> str | None:
+    """Extract DistrictId from the initial server-selection TCP response.
+
+    This response is a length-prefixed FlatBuffers message. DistrictId is at
+    byte 96 in the current schema. Validate the frame and its following server
+    address before accepting the value so an arbitrary integer at that offset
+    is not mistaken for a server ID.
+    """
+    if len(payload) < 168:
+        return None
+
+    frame_size = struct.unpack_from("<I", payload, 0)[0] + 4
+    if frame_size > len(payload) or frame_size < 168:
+        return None
+    if struct.unpack_from("<I", payload, 4)[0] != 20:
+        return None
+
+    server_id = struct.unpack_from("<I", payload, 96)[0]
+    if not 20_000 <= server_id <= 99_999:
+        return None
+
+    address_length = struct.unpack_from("<I", payload, 132)[0]
+    address_end = 136 + address_length
+    if not 1 <= address_length <= 255 or address_end > frame_size:
+        return None
+    try:
+        address = payload[136:address_end].decode("ascii")
+        ipaddress.ip_address(address)
+    except (UnicodeDecodeError, ValueError):
+        return None
+
+    return str(server_id)

src/nte_history_exporter/decoder/structured_protocol.py

@@ -0,0 +1,547 @@
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass, field, replace
+from datetime import datetime, timezone
+from typing import Literal
+
+
+RecordType = Literal["monopoly", "fork"]
+
+MONOPOLY_MARKER = b"FMonopolyLotteryRecordData"
+FORK_MARKER = b"FForkLotteryRecordData"
+MAX_ROWS_PER_BLOCK = 100
+MAX_STRING_LENGTH = 256
+DOTNET_EPOCH_TICKS = 621_355_968_000_000_000
+DOTNET_TICKS_PER_SECOND = 10_000_000
+MIN_UNIX_SECONDS = 1_500_000_000
+MAX_UNIX_SECONDS = 4_102_444_800
+PROTOCOL_CONSTANT = 0x03000000
+MONOPOLY_BLOCK_KIND = 527
+FORK_BLOCK_KIND = 5906
+MONOPOLY_ENVELOPE_FOOTER = 1_774_080
+
+
+class StructuredProtocolError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class StructuredRecord:
+    record_type: RecordType
+    item_id: str
+    count: int
+    ticks: int
+    timestamp_unix: float
+    timestamp_decoded: str
+    pool_id: str | None
+    roll_points_raw: int | None
+    secondary_item_id: str | None
+    secondary_count: int | None
+    record_start: int
+    record_end: int
+    record_hex: str
+    protocol_view: str
+    source_index: int | None = None
+    generation_index: int | None = None
+
+
+@dataclass(frozen=True)
+class ProtocolEnvelope:
+    record_type: RecordType
+    stream_key: str
+    page_index: int
+    query_high: bool
+    segment_index: int
+
+
+@dataclass(frozen=True)
+class StructuredBlock:
+    record_type: RecordType
+    marker_offset: int
+    declared_size: int
+    rows: tuple[StructuredRecord, ...]
+    envelope: ProtocolEnvelope | None
+
+
+@dataclass
+class _Generation:
+    index: int
+    segments: dict[int, StructuredBlock] = field(default_factory=dict)
+
+
+@dataclass
+class _Stream:
+    generations: list[_Generation] = field(default_factory=list)
+
+
+class StructuredProtocolAssembler:
+    """Assemble retransmitted and overlapping structured snapshot segments."""
+
+    def __init__(self) -> None:
+        self._stream_order: list[str] = []
+        self._streams: dict[str, _Stream] = {}
+        self._legacy_rows: list[StructuredRecord] = []
+        self.warnings: list[dict[str, str | int]] = []
+
+    def add_blocks(self, blocks: list[StructuredBlock]) -> None:
+        for block in blocks:
+            self.add_block(block)
+
+    def add_block(self, block: StructuredBlock) -> bool:
+        envelope = block.envelope
+        if envelope is None:
+            if not self._legacy_rows:
+                self._stream_order.append("__legacy__")
+            self._legacy_rows.extend(block.rows)
+            return True
+
+        stream = self._streams.get(envelope.stream_key)
+        if stream is None:
+            stream = _Stream()
+            self._streams[envelope.stream_key] = stream
+            self._stream_order.append(envelope.stream_key)
+        if not stream.generations:
+            stream.generations.append(_Generation(0))
+        generation = stream.generations[-1]
+        existing = generation.segments.get(envelope.segment_index)
+        if existing is not None:
+            if _block_signature(existing) == _block_signature(block):
+                return False
+            generation = _Generation(len(stream.generations))
+            stream.generations.append(generation)
+        generation.segments[envelope.segment_index] = block
+        return True
+
+    def rows(self, record_type: RecordType | None = None) -> list[StructuredRecord]:
+        rows: list[StructuredRecord] = []
+        for stream_key in self._stream_order:
+            if stream_key == "__legacy__":
+                rows.extend(
+                    row
+                    for row in self._legacy_rows
+                    if record_type is None or row.record_type == record_type
+                )
+                continue
+            assembled = self._assemble_stream(stream_key, self._streams[stream_key])
+            rows.extend(row for row in assembled if record_type is None or row.record_type == record_type)
+        return rows
+
+    def _assemble_stream(self, stream_key: str, stream: _Stream) -> list[StructuredRecord]:
+        result: list[StructuredRecord] = []
+        result_max_segment: int | None = None
+        for generation in stream.generations:
+            if not generation.segments:
+                continue
+            generation_rows = _generation_rows(generation)
+            segment_indexes = sorted(generation.segments)
+            generation_min = segment_indexes[0]
+            generation_max = segment_indexes[-1]
+            if not result:
+                result = generation_rows
+                result_max_segment = generation_max
+                continue
+            if generation_min == 0:
+                if result_max_segment is None or generation_max >= result_max_segment:
+                    result = generation_rows
+                    result_max_segment = generation_max
+                    continue
+                merged = _partial_snapshot_merge(generation_rows, result)
+                if merged is not None:
+                    result = merged
+                else:
+                    self._warn(stream_key, generation, "partial snapshot cannot be merged safely")
+                continue
+            if result_max_segment is not None and generation_min > result_max_segment:
+                result.extend(generation_rows)
+                result_max_segment = generation_max
+                continue
+            self._warn(stream_key, generation, "non-zero snapshot reset cannot be merged safely")
+        return result
+
+    def _warn(self, stream_key: str, generation: _Generation, message: str) -> None:
+        warning = {
+            "code": "AMBIGUOUS_STRUCTURED_SNAPSHOT",
+            "stream_key": stream_key,
+            "generation_index": generation.index,
+            "message": message,
+        }
+        if warning not in self.warnings:
+            self.warnings.append(warning)
+
+
+def parse_structured_records(payload: bytes, record_type: RecordType) -> list[StructuredRecord]:
+    """Parse typed history blocks from raw or bit-shifted protocol payloads.
+
+    Invalid candidates are ignored deliberately: callers use this parser only
+    as enrichment/fallback and retain the established decoder as their primary
+    path.
+    """
+    assembler = StructuredProtocolAssembler()
+    assembler.add_blocks(parse_structured_blocks(payload, record_type))
+    return assembler.rows(record_type)
+
+
+def parse_structured_blocks(
+    payload: bytes,
+    record_type: RecordType,
+    *,
+    source_index: int | None = None,
+) -> list[StructuredBlock]:
+    marker = MONOPOLY_MARKER if record_type == "monopoly" else FORK_MARKER
+    for view_name, data in _iter_protocol_views(payload):
+        if marker not in data:
+            continue
+        blocks: list[StructuredBlock] = []
+        search_from = 0
+        while True:
+            marker_pos = data.find(marker, search_from)
+            if marker_pos < 0:
+                break
+            try:
+                parsed = _parse_block(
+                    data, marker_pos, record_type, marker, view_name, source_index
+                )
+            except (StructuredProtocolError, UnicodeDecodeError):
+                parsed = None
+            if parsed is not None:
+                blocks.append(parsed)
+            search_from = marker_pos + len(marker)
+        if blocks:
+            return blocks
+    return []
+
+
+def _parse_block(
+    data: bytes,
+    marker_pos: int,
+    record_type: RecordType,
+    marker: bytes,
+    view_name: str,
+    source_index: int | None,
+) -> StructuredBlock:
+    envelope = _parse_protocol_envelope(record_type, data, marker_pos, view_name)
+    pos = marker_pos + len(marker)
+    if _byte_at(data, pos) == 0:
+        pos += 1
+    _reserved = _u32_at(data, pos)
+    declared_size = _u32_at(data, pos + 4)
+    row_count = _u32_at(data, pos + 8)
+    pos += 12
+    if row_count > MAX_ROWS_PER_BLOCK:
+        raise StructuredProtocolError(f"row count is too large: {row_count}")
+    if declared_size > len(data) - pos:
+        raise StructuredProtocolError("declared block size exceeds payload")
+
+    reader = _Reader(data, pos)
+    records = []
+    for _row_index in range(row_count):
+        row_start = reader.pos
+        if record_type == "monopoly":
+            record = _parse_monopoly_row(reader, row_start, view_name, source_index)
+        else:
+            record = _parse_fork_row(reader, row_start, view_name, source_index)
+        records.append(record)
+    return StructuredBlock(
+        record_type=record_type,
+        marker_offset=marker_pos,
+        declared_size=declared_size,
+        rows=tuple(records),
+        envelope=envelope,
+    )
+
+
+def _parse_monopoly_row(
+    reader: "_Reader", row_start: int, view_name: str, source_index: int | None
+) -> StructuredRecord:
+    roll_points_raw = reader.u32()
+    item_spec = reader.string()
+    _reserved = reader.u32()
+    secondary_count = reader.u32()
+    secondary_item_id = reader.string()
+    result_or_pool = reader.string()
+
+    pool_pos = reader.pos
+    possible_pool = reader.try_string()
+    if possible_pool and possible_pool.startswith("CardPool_"):
+        pool_id = possible_pool
+    else:
+        reader.pos = pool_pos
+        pool_id = result_or_pool if result_or_pool.startswith("CardPool_") else None
+
+    ticks = reader.u64()
+    return _make_record(
+        reader,
+        "monopoly",
+        item_spec,
+        ticks,
+        pool_id,
+        roll_points_raw,
+        secondary_item_id or None,
+        secondary_count,
+        row_start,
+        view_name,
+        source_index,
+    )
+
+
+def _parse_fork_row(
+    reader: "_Reader", row_start: int, view_name: str, source_index: int | None
+) -> StructuredRecord:
+    item_spec = reader.string()
+    pool_id = reader.string()
+    ticks = reader.u64()
+    return _make_record(
+        reader,
+        "fork",
+        item_spec,
+        ticks,
+        pool_id or None,
+        None,
+        None,
+        None,
+        row_start,
+        view_name,
+        source_index,
+    )
+
+
+def _make_record(
+    reader: "_Reader",
+    record_type: RecordType,
+    item_spec: str,
+    ticks: int,
+    pool_id: str | None,
+    roll_points_raw: int | None,
+    secondary_item_id: str | None,
+    secondary_count: int | None,
+    row_start: int,
+    view_name: str,
+    source_index: int | None,
+) -> StructuredRecord:
+    item_id, count = _parse_item_spec(item_spec)
+    if not item_id:
+        raise StructuredProtocolError("structured item ID is empty")
+    timestamp_unix = (ticks - DOTNET_EPOCH_TICKS) / DOTNET_TICKS_PER_SECOND
+    if not MIN_UNIX_SECONDS <= timestamp_unix <= MAX_UNIX_SECONDS:
+        raise StructuredProtocolError("structured timestamp is out of range")
+    timestamp_decoded = datetime.fromtimestamp(timestamp_unix, timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
+    return StructuredRecord(
+        record_type=record_type,
+        item_id=item_id,
+        count=count,
+        ticks=ticks,
+        timestamp_unix=timestamp_unix,
+        timestamp_decoded=timestamp_decoded,
+        pool_id=pool_id,
+        roll_points_raw=roll_points_raw,
+        secondary_item_id=secondary_item_id,
+        secondary_count=secondary_count,
+        record_start=row_start,
+        record_end=reader.pos,
+        record_hex=reader.data[row_start : reader.pos].hex(),
+        protocol_view=view_name,
+        source_index=source_index,
+    )
+
+
+def _parse_item_spec(value: str) -> tuple[str, int]:
+    item_id, separator, raw_count = value.rpartition(",")
+    if separator:
+        try:
+            count = int(raw_count)
+        except ValueError:
+            count = 0
+        if item_id and count > 0:
+            return item_id, count
+    return value, 1
+
+
+def _parse_protocol_envelope(
+    record_type: RecordType,
+    data: bytes,
+    marker_pos: int,
+    view_name: str,
+) -> ProtocolEnvelope | None:
+    if marker_pos == 0 or not view_name.startswith("shift8:"):
+        return None
+    if record_type == "monopoly":
+        if marker_pos < 26:
+            raise StructuredProtocolError("monopoly envelope is truncated")
+        protocol_constant = _relative_u32(data, marker_pos, -26)
+        query_raw = _relative_u32(data, marker_pos, -22)
+        page_raw = _relative_u32(data, marker_pos, -18)
+        block_kind = _relative_u32(data, marker_pos, -14)
+        pool_token = _relative_u32(data, marker_pos, -10)
+        footer = _relative_u32(data, marker_pos, -6)
+        if (
+            protocol_constant != PROTOCOL_CONSTANT
+            or block_kind != MONOPOLY_BLOCK_KIND
+            or footer != MONOPOLY_ENVELOPE_FOOTER
+        ):
+            raise StructuredProtocolError("invalid monopoly envelope constants")
+        stream_key = f"monopoly:{pool_token}"
+    else:
+        if marker_pos < 17:
+            raise StructuredProtocolError("fork envelope is truncated")
+        protocol_constant = _relative_u32(data, marker_pos, -17)
+        query_raw = _relative_u32(data, marker_pos, -13)
+        page_raw = _relative_u32(data, marker_pos, -9)
+        block_kind = _relative_u32(data, marker_pos, -5)
+        if protocol_constant != PROTOCOL_CONSTANT or block_kind != FORK_BLOCK_KIND:
+            raise StructuredProtocolError("invalid fork envelope constants")
+        stream_key = "fork"
+    page_index = page_raw & 0x7FFFFFFF
+    query_high = bool(query_raw & 0x80000000)
+    return ProtocolEnvelope(
+        record_type=record_type,
+        stream_key=stream_key,
+        page_index=page_index,
+        query_high=query_high,
+        segment_index=_segment_index(page_index, query_high),
+    )
+
+
+def _segment_index(page_index: int, query_high: bool) -> int:
+    if query_high:
+        return page_index * 2
+    if page_index > 0:
+        return page_index * 2 - 1
+    raise StructuredProtocolError("low query cannot describe page zero")
+
+
+def _row_signature(row: StructuredRecord) -> tuple:
+    return (
+        row.record_type,
+        row.ticks,
+        row.pool_id,
+        row.item_id,
+        row.count,
+        row.roll_points_raw,
+        row.secondary_item_id,
+        row.secondary_count,
+    )
+
+
+def _block_signature(block: StructuredBlock) -> tuple:
+    return block.record_type, tuple(_row_signature(row) for row in block.rows)
+
+
+def _generation_rows(generation: _Generation) -> list[StructuredRecord]:
+    rows = []
+    for segment_index in sorted(generation.segments):
+        block = generation.segments[segment_index]
+        rows.extend(replace(row, generation_index=generation.index) for row in block.rows)
+    return rows
+
+
+def _partial_snapshot_merge(
+    new_rows: list[StructuredRecord], old_rows: list[StructuredRecord]
+) -> list[StructuredRecord] | None:
+    if not new_rows:
+        return list(old_rows)
+    if not old_rows:
+        return list(new_rows)
+    new_signatures = [_row_signature(row) for row in new_rows]
+    old_signatures = [_row_signature(row) for row in old_rows]
+    max_overlap = min(len(new_signatures), len(old_signatures))
+    matches: list[tuple[int, int]] = []
+    for overlap in range(max_overlap, 0, -1):
+        suffix = new_signatures[-overlap:]
+        for position in range(len(old_signatures) - overlap + 1):
+            if old_signatures[position : position + overlap] == suffix:
+                matches.append((overlap, position))
+        if matches:
+            break
+    if len(matches) != 1:
+        return None
+    overlap, position = matches[0]
+    return [*new_rows, *old_rows[position + overlap :]]
+
+
+class _Reader:
+    def __init__(self, data: bytes, pos: int) -> None:
+        self.data = data
+        self.pos = pos
+
+    def u32(self) -> int:
+        value = _u32_at(self.data, self.pos)
+        self.pos += 4
+        return value
+
+    def u64(self) -> int:
+        value = _u64_at(self.data, self.pos)
+        self.pos += 8
+        return value
+
+    def string(self) -> str:
+        length_pos = self.pos
+        length = self.u32()
+        if length == 0 or length > MAX_STRING_LENGTH:
+            raise StructuredProtocolError(f"invalid string length {length} at {length_pos}")
+        end = self.pos + length
+        raw = self.data[self.pos:end]
+        if len(raw) != length:
+            raise StructuredProtocolError("string exceeds payload")
+        self.pos = end
+        if raw.endswith(b"\0"):
+            raw = raw[:-1]
+        return raw.decode("utf-8")
+
+    def try_string(self) -> str | None:
+        start = self.pos
+        try:
+            return self.string()
+        except (StructuredProtocolError, UnicodeDecodeError):
+            self.pos = start
+            return None
+
+
+def _iter_protocol_views(payload: bytes):
+    yield "raw", payload
+    for bit_shift in range(1, 8):
+        shifted = _decode_shifted_bytes(payload, byte_offset=8, bit_shift=bit_shift)
+        yield f"shift8:{bit_shift}", shifted
+
+
+def _decode_shifted_bytes(data: bytes, *, byte_offset: int, bit_shift: int) -> bytes:
+    result = bytearray()
+    count = max(0, len(data) - byte_offset)
+    for index in range(count):
+        bit_pos = (byte_offset + index) * 8 + bit_shift
+        byte_pos, shift = divmod(bit_pos, 8)
+        if byte_pos >= len(data):
+            break
+        value = data[byte_pos] >> shift
+        if shift and byte_pos + 1 < len(data):
+            value |= data[byte_pos + 1] << (8 - shift)
+        result.append(value & 0xFF)
+    return bytes(result)
+
+
+def _byte_at(data: bytes, pos: int) -> int:
+    try:
+        return data[pos]
+    except IndexError as exc:
+        raise StructuredProtocolError("byte exceeds payload") from exc
+
+
+def _u32_at(data: bytes, pos: int) -> int:
+    try:
+        return struct.unpack_from("<I", data, pos)[0]
+    except struct.error as exc:
+        raise StructuredProtocolError("u32 exceeds payload") from exc
+
+
+def _relative_u32(data: bytes, marker_pos: int, relative_pos: int) -> int:
+    pos = marker_pos + relative_pos
+    if pos < 0:
+        raise StructuredProtocolError("envelope position precedes payload")
+    return _u32_at(data, pos)
+
+
+def _u64_at(data: bytes, pos: int) -> int:
+    try:
+        return struct.unpack_from("<Q", data, pos)[0]
+    except struct.error as exc:
+        raise StructuredProtocolError("u64 exceeds payload") from exc

src/nte_history_exporter/export/csv_export.py

@@ -27,11 +27,19 @@ FIELDNAMES = [
     "result_source_raw",
     "dice",
     "dice_raw_u32",
+    "decoder_mode",
+    "structured_protocol_view",
+    "structured_generation_index",
+    "structured_pool_id",
+    "structured_assembly",
+    "structured_assembly_warning_count",
     "reward_type",
     "reward_id",
     "reward_name",
     "reward_rank",
     "quantity",
+    "secondary_reward_id",
+    "secondary_quantity",
     "timestamp_decoded",
     "timestamp_raw_hex",
     "timestamp_ticks",
@@ -50,6 +58,7 @@ FIELDNAMES = [
     "record_len",
     "dice_offset_in_record",
     "record_hex",
+    "record_flag",
 ]
 
 

src/nte_history_exporter/export/json_export.py

@@ -9,7 +9,9 @@ from nte_history_exporter.constants import (
     EXPORTER_NAME,
     GAME_NAME,
     POOL_META,
+    MYSTERY_BOX_BANNER_ID,
 )
+from nte_history_exporter.decoder.server_region import account_region_for_server
 
 
 def build_export_json(
@@ -19,6 +21,7 @@ def build_export_json(
     source: str = "packet_capture",
     capture_source: str | None = None,
     user_uid: str | None = None,
+    server_id: str | None = None,
     flow_index: int | None = None,
     candidate_request_response_pairs: int | None = None,
     pages_seen: list[int] | None = None,
@@ -43,6 +46,7 @@ def build_export_json(
         scan["pages_seen"] = pages_seen
 
     normalized_user_uid = user_uid.strip() if user_uid else ""
+    normalized_server_id = str(server_id).strip() if server_id else ""
     export: dict[str, Any] = {
         "format": "nte-history-export",
         "format_version": 1,
@@ -65,6 +69,11 @@ def build_export_json(
     )
     if normalized_user_uid:
         export["user_uid"] = normalized_user_uid
+    if normalized_server_id:
+        export["server_id"] = normalized_server_id
+        account_region = account_region_for_server(normalized_server_id)
+        if account_region:
+            export["account_region"] = account_region
     export["records"] = [_record_for_export(r) for r in exported]
     return export
 
@@ -83,6 +92,21 @@ def _record_for_export(row: dict[str, Any]) -> dict[str, Any]:
             "source_type": row.get("source_type"),
         }
 
+    if row.get("pool_group_id") == MYSTERY_BOX_BANNER_ID:
+        return {
+            "uid": row.get("uid"),
+            "pool_group_id": row.get("pool_group_id"),
+            "timestamp": row.get("timestamp_decoded"),
+            "timestamp_group_ordinal": row.get("timestamp_group_ordinal"),
+            "result_type": "single_pull",
+            "reward_type": row.get("reward_type"),
+            "reward_id": row.get("reward_id"),
+            "reward_name": row.get("reward_name"),
+            "reward_rank": row.get("reward_rank"),
+            "quantity": row.get("quantity"),
+            "source_type": row.get("source_type"),
+        }
+
     return {
         "uid": row.get("uid"),
         "pool_group_id": row.get("pool_group_id", BANNER_ID),

src/nte_history_exporter/live_capture/diagnostics.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Iterable
+
+
+DIAGNOSTIC_FORMAT = "nte-capture-diagnostics"
+DIAGNOSTIC_FORMAT_VERSION = 1
+MAX_EVENTS = 200
+
+
+class CaptureDiagnostics:
+    """Collect bounded, privacy-safe observations about a capture session."""
+
+    def __init__(self) -> None:
+        self.counters: Counter[str] = Counter()
+        self.event_counts: Counter[str] = Counter()
+        self.reason_counts: Counter[str] = Counter()
+        self.events: list[dict[str, Any]] = []
+
+    def observe_packet(self, protocol: str) -> None:
+        self.counters["packets_seen"] += 1
+        if protocol == "udp":
+            self.counters["udp_packets_seen"] += 1
+        else:
+            self.counters["non_udp_packets_ignored"] += 1
+
+    def add_event(
+        self,
+        code: str,
+        packet_index: int,
+        *,
+        reason: bool = False,
+        **fields: Any,
+    ) -> None:
+        self.event_counts[code] += 1
+        if reason:
+            self.reason_counts[code] += 1
+        if len(self.events) >= MAX_EVENTS:
+            self.counters["events_omitted"] += 1
+            return
+        event: dict[str, Any] = {"packet_index": packet_index, "code": code}
+        event.update({key: value for key, value in fields.items() if value is not None})
+        self.events.append(event)
+
+    def report(self, pending_requests: Iterable[Any]) -> dict[str, Any]:
+        pending = [
+            {
+                "kind": request.kind,
+                "page": request.page,
+                "response_candidates": request.response_candidates,
+                "response_candidate_lengths": list(request.response_candidate_lengths),
+            }
+            for request in pending_requests
+        ]
+        return {
+            "format": DIAGNOSTIC_FORMAT,
+            "format_version": DIAGNOSTIC_FORMAT_VERSION,
+            "privacy": {
+                "contains_network_addresses": False,
+                "contains_network_ports": False,
+                "contains_packet_timestamps": False,
+                "contains_payload_bytes": False,
+                "contains_user_uid": False,
+            },
+            "counters": dict(sorted(self.counters.items())),
+            "event_counts": dict(sorted(self.event_counts.items())),
+            "reason_counts": dict(sorted(self.reason_counts.items())),
+            "events": list(self.events),
+            "pending_requests": pending,
+        }
+
+
+def new_diagnostics_path(output_dir: str | Path = "exports") -> Path:
+    directory = Path(output_dir)
+    directory.mkdir(parents=True, exist_ok=True)
+    stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    path = directory / f"Capture_{stamp}.diagnostics.json"
+    counter = 2
+    while path.exists():
+        path = directory / f"Capture_{stamp}_{counter}.diagnostics.json"
+        counter += 1
+    return path
+
+
+def write_capture_diagnostics(path: str | Path, report: dict[str, Any]) -> None:
+    Path(path).write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")

src/nte_history_exporter/live_capture/runner.py

@@ -13,6 +13,7 @@ from nte_history_exporter.decoder.boundary import annotate_groups, select_contin
 from nte_history_exporter.export.csv_export import write_csv
 from nte_history_exporter.export.json_export import build_export_json
 from nte_history_exporter.live_capture.backends import open_capture_backend
+from nte_history_exporter.live_capture.diagnostics import new_diagnostics_path, write_capture_diagnostics
 from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket
 from nte_history_exporter.live_capture.stop_key import StopKeyMonitor
 from nte_history_exporter.live_capture.windows_raw import detect_local_ipv4
@@ -22,6 +23,7 @@ EXPORT_PREFIXES = {
     "permanent": "Permanent",
     "limited_character": "Limited",
     "arc_miracle_box": "Arc",
+    "mystery_box": "MysteryBox",
 }
 
 CAPTURE_SOURCE_LABELS = {
@@ -135,15 +137,22 @@ def run_live_capture(
         )
 
     exports = []
+    diagnostics_path = None
+    if write_debug_csv:
+        diagnostics_path = new_diagnostics_path()
+        write_capture_diagnostics(diagnostics_path, session.diagnostic_report())
     resolved_user_uid = user_uid or session.user_uid
     if session.kinds_seen() and not resolved_user_uid:
         resolved_user_uid = console.prompt_user_uid()
+    resolved_server_id = session.server_id
+    if session.kinds_seen() and not resolved_server_id:
+        resolved_server_id = console.prompt_server_id()
     capture_source = CAPTURE_SOURCE_LABELS.get(capture.name, capture.name)
     for kind in session.kinds_seen():
         pairs = session.pairs_for_kind(kind)
         best_run, run_warnings = select_continuous_run_from_page_1(pairs)
         rows = session.build_rows(kind)
-        if kind != "arc_miracle_box":
+        if kind not in {"arc_miracle_box", "mystery_box"}:
             rows = annotate_groups(rows)
         warnings = run_warnings
         pages_seen = [p[0] for p in best_run]
@@ -156,6 +165,7 @@ def run_live_capture(
             source="live_capture",
             capture_source=capture_source,
             user_uid=resolved_user_uid,
+            server_id=resolved_server_id,
             pages_seen=pages_seen,
         )
         payload = json.dumps(export, ensure_ascii=False, indent=2)
@@ -164,6 +174,7 @@ def run_live_capture(
             {
                 "kind": kind,
                 "csv_path": csv_path if write_debug_csv else None,
+                "diagnostics_path": diagnostics_path if write_debug_csv else None,
                 "json_path": json_path,
                 "export": export,
                 "payload": payload,
@@ -176,7 +187,9 @@ def run_live_capture(
         console.print_note("Make sure the capture backend is running, then reopen the")
         console.print_note("history screen and scroll from page 1. If no page messages")
         console.print_note("appear, return to the main menu and re-enter the game.")
-        return {"exports": []}
+        if diagnostics_path is not None:
+            console.print_note(f"Diagnostics written: {diagnostics_path}")
+        return {"exports": [], "diagnostics_path": diagnostics_path}
 
     for item in exports:
         scan = item["export"]["scan"]
@@ -194,6 +207,8 @@ def run_live_capture(
         if item["csv_path"] is not None:
             console.print_note(f"CSV written: {item['csv_path']}")
         console.print_note(f"Export written: {item['json_path']}")
+    if diagnostics_path is not None:
+        console.print_note(f"Diagnostics written: {diagnostics_path}")
 
     if copy_clipboard and len(exports) == 1:
         if copy_to_clipboard(exports[0]["payload"]):
@@ -203,7 +218,7 @@ def run_live_capture(
     elif copy_clipboard and len(exports) > 1:
         console.print_note("Multiple banners captured; clipboard copy skipped so one export")
         console.print_note("does not overwrite another.")
-    return {"exports": exports}
+    return {"exports": exports, "diagnostics_path": diagnostics_path}
 
 
 def export_paths(kind: str, user_uid: str | None = None) -> tuple[Path, Path]:

src/nte_history_exporter/live_capture/session.py

@@ -17,9 +17,21 @@ from nte_history_exporter.decoder.protocol import (
     history_request_kind,
     is_history_request,
     request_page,
+    response_contains_history_marker,
 )
 from nte_history_exporter.decoder.run import build_rows_from_pairs
+from nte_history_exporter.decoder.mystery_box import (
+    build_mystery_box_rows_from_pairs,
+    is_mystery_box_history_request,
+    mystery_box_request_page,
+    parse_mystery_box_response,
+    select_continuous_mystery_box_run,
+)
+from nte_history_exporter.decoder.structured_protocol import FORK_MARKER, MONOPOLY_MARKER
+from nte_history_exporter.constants import MYSTERY_BOX_MARKER
 from nte_history_exporter.decoder.user_uid import extract_user_uid_candidates
+from nte_history_exporter.decoder.server_region import extract_server_id
+from nte_history_exporter.live_capture.diagnostics import CaptureDiagnostics
 
 
 @dataclass
@@ -61,6 +73,8 @@ class LiveHistorySession:
         self.unanswered_pages: dict[str, dict[int, str]] = {}
         self.user_uid: str | None = None
         self.user_uid_candidates: Counter[str] = Counter()
+        self.server_id: str | None = None
+        self.diagnostics = CaptureDiagnostics()
 
     def _mark_unanswered(self, request: PendingRequest) -> None:
         if request.response_candidates:
@@ -89,6 +103,14 @@ class LiveHistorySession:
             )
             if same_stream and (request.page == 1 or pending.page == request.page):
                 self._mark_unanswered(pending)
+                self.diagnostics.add_event(
+                    "REQUEST_REPLACED",
+                    request.request_msg,
+                    reason=True,
+                    kind=pending.kind,
+                    page=pending.page,
+                    response_candidates=pending.response_candidates,
+                )
             else:
                 retained.append(pending)
         self.pending = retained
@@ -97,10 +119,16 @@ class LiveHistorySession:
 
     def process_packet(self, packet: UdpPacket) -> bool:
         self.packet_count += 1
+        self.diagnostics.observe_packet(packet.protocol)
         candidates = extract_user_uid_candidates(packet.payload)
         if candidates:
             self.user_uid_candidates.update(candidates)
             self.user_uid = self.user_uid_candidates.most_common(1)[0][0]
+        if packet.protocol == "tcp" and packet.dst_ip == self.local_ip and not self.server_id:
+            self.server_id = extract_server_id(packet.payload)
+            if self.server_id:
+                self.diagnostics.counters["server_ids_detected"] += 1
+                self.diagnostics.add_event("SERVER_ID_DETECTED", self.packet_count)
         if packet.protocol != "udp":
             return False
 
@@ -118,6 +146,13 @@ class LiveHistorySession:
                 dst_port=packet.dst_port,
             )
             self._queue_request(req)
+            self.diagnostics.counters["history_requests_recognized"] += 1
+            self.diagnostics.add_event(
+                "HISTORY_REQUEST_RECOGNIZED",
+                self.packet_count,
+                kind=req.kind,
+                page=req.page,
+            )
             self.last_page_seen = req.page
             return False
 
@@ -135,10 +170,41 @@ class LiveHistorySession:
                 dst_port=packet.dst_port,
             )
             self._queue_request(req)
+            self.diagnostics.counters["history_requests_recognized"] += 1
+            self.diagnostics.add_event(
+                "HISTORY_REQUEST_RECOGNIZED",
+                self.packet_count,
+                kind=req.kind,
+                page=req.page,
+            )
             self.last_page_seen = req.page
             return False
 
-        if packet.dst_ip != self.local_ip or len(packet.payload) < 100:
+        if packet.src_ip == self.local_ip and is_mystery_box_history_request(packet.payload):
+            page = mystery_box_request_page(packet.payload)
+            req = PendingRequest(
+                page=page,
+                offset=page * 2,
+                kind="mystery_box",
+                request_msg=self.packet_count,
+                request_time=packet.timestamp,
+                src_ip=packet.src_ip,
+                dst_ip=packet.dst_ip,
+                src_port=packet.src_port,
+                dst_port=packet.dst_port,
+            )
+            self._queue_request(req)
+            self.diagnostics.counters["history_requests_recognized"] += 1
+            self.diagnostics.add_event(
+                "HISTORY_REQUEST_RECOGNIZED",
+                self.packet_count,
+                kind=req.kind,
+                page=req.page,
+            )
+            self.last_page_seen = req.page
+            return False
+
+        if packet.dst_ip != self.local_ip:
             return False
 
         connection_candidates = [
@@ -154,27 +220,69 @@ class LiveHistorySession:
         if not connection_candidates:
             return False
 
+        if len(packet.payload) < 100:
+            self._record_rejected_candidate(
+                connection_candidates,
+                packet.payload,
+                "RESPONSE_TOO_SHORT",
+            )
+            return False
+
         monopoly_records = decode_response_records(packet.payload)
         arc_records = parse_arc_response(packet.payload) if not monopoly_records else []
+        mystery_box_records = (
+            parse_mystery_box_response(packet.payload)
+            if not monopoly_records and not arc_records
+            else []
+        )
         if monopoly_records:
-            candidates = [req for req in connection_candidates if req.kind != "arc_miracle_box"]
+            candidates = [
+                req
+                for req in connection_candidates
+                if req.kind not in {"arc_miracle_box", "mystery_box"}
+            ]
             records = monopoly_records
         elif arc_records:
             candidates = [req for req in connection_candidates if req.kind == "arc_miracle_box"]
             records = arc_records
+        elif mystery_box_records:
+            candidates = [req for req in connection_candidates if req.kind == "mystery_box"]
+            records = mystery_box_records
         else:
             candidates = connection_candidates
             records = []
 
-        if not records or not candidates:
+        if records and not candidates:
-            for req in candidates:
+            self._record_rejected_candidate(
-                req.response_candidates += 1
+                connection_candidates,
-                req.response_candidate_lengths = (
+                packet.payload,
-                    *req.response_candidate_lengths[-4:],
+                "RESPONSE_KIND_MISMATCH",
-                    len(packet.payload),
+            )
-                )
             return False
 
+        if not records:
+            reason = (
+                "HISTORY_MARKER_PARSE_FAILED"
+                if response_contains_history_marker(packet.payload)
+                or MONOPOLY_MARKER in packet.payload
+                or FORK_MARKER in packet.payload
+                or MYSTERY_BOX_MARKER in packet.payload
+                else "NO_HISTORY_MARKER"
+            )
+            self._record_rejected_candidate(candidates, packet.payload, reason)
+            return False
+
+        decoder_modes = sorted({record.get("decoder_mode", "heuristic") for record in records})
+        self.diagnostics.counters["history_responses_decoded"] += 1
+        self.diagnostics.add_event(
+            "HISTORY_RESPONSE_DECODED",
+            self.packet_count,
+            kind=candidates[0].kind,
+            payload_length=len(packet.payload),
+            record_count=len(records),
+            decoder_modes=decoder_modes,
+        )
+
         page_count = max(1, (len(records) + 4) // 5)
         if len(records) < 5:
             selected = [candidates[-1]]
@@ -208,9 +316,43 @@ class LiveHistorySession:
             )
             self.last_match_time = packet.timestamp
             self.last_page_seen = req.page
+            self.diagnostics.counters["pages_matched"] += 1
+            self.diagnostics.add_event(
+                "PAGE_RESPONSE_MATCHED",
+                self.packet_count,
+                kind=req.kind,
+                page=req.page,
+                record_count=slice_count,
+            )
 
         return bool(selected)
 
+    def _record_rejected_candidate(
+        self,
+        requests: list[PendingRequest],
+        payload: bytes,
+        reason: str,
+    ) -> None:
+        for req in requests:
+            req.response_candidates += 1
+            req.response_candidate_lengths = (
+                *req.response_candidate_lengths[-4:],
+                len(payload),
+            )
+        self.diagnostics.counters["response_candidates_rejected"] += 1
+        self.diagnostics.add_event(
+            reason,
+            self.packet_count,
+            reason=True,
+            kind=requests[0].kind if requests else None,
+            page=requests[0].page if requests else None,
+            payload_length=len(payload),
+            matching_requests=len(requests),
+        )
+
+    def diagnostic_report(self) -> dict[str, Any]:
+        return self.diagnostics.report(self.pending)
+
     def kinds_seen(self) -> list[str]:
         seen = []
         for pair in self.pairs:
@@ -250,6 +392,9 @@ class LiveHistorySession:
         if kind == "arc_miracle_box":
             best_run, _warnings = select_continuous_arc_run(self.pairs_for_kind(kind))
             return build_arc_rows_from_pairs(best_run)
+        if kind == "mystery_box":
+            best_run, _warnings = select_continuous_mystery_box_run(self.pairs_for_kind(kind))
+            return build_mystery_box_rows_from_pairs(best_run)
         return build_rows_from_pairs(self.best_run(kind))
 
     def best_run(self, kind: str | None = None) -> list[tuple]:

src/nte_history_exporter/live_capture/stop_key.py

@@ -5,6 +5,41 @@ import select
 import sys
 
 
+def _read_windows_key() -> None:
+    import msvcrt
+
+    key = msvcrt.getch()
+    # Function and arrow keys are emitted as a two-byte sequence. Consume the
+    # second byte so it cannot immediately satisfy the next key prompt.
+    if key in {b"\x00", b"\xe0"}:
+        msvcrt.getch()
+
+
+def wait_for_keypress() -> None:
+    """Wait for one key on an interactive terminal, or Enter on redirected input."""
+    if os.name == "nt" and sys.stdin.isatty():
+        _read_windows_key()
+        return
+
+    if not sys.stdin.isatty():
+        try:
+            input()
+        except EOFError:
+            pass
+        return
+
+    import termios
+    import tty
+
+    fd = sys.stdin.fileno()
+    original_terminal = termios.tcgetattr(fd)
+    try:
+        tty.setcbreak(fd)
+        os.read(fd, 1)
+    finally:
+        termios.tcsetattr(fd, termios.TCSADRAIN, original_terminal)
+
+
 class StopKeyMonitor:
     def __init__(self) -> None:
         self._windows = os.name == "nt"
@@ -29,7 +64,7 @@ class StopKeyMonitor:
 
             if not msvcrt.kbhit():
                 return False
-            msvcrt.getch()
+            _read_windows_key()
             return True
 
         if self._fd is None:

src/nte_history_exporter/mapping_update.py

@@ -0,0 +1,422 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+NTE_ASSETS_REPOSITORY = "https://github.com/Waifus-Grace/NTE_Assets"
+DEFAULT_SOURCE_REF = "main"
+ASSET_TABLES = {
+    "characters": "DataTable/Character/DT_Character.json",
+    "arcs": "DataTable/Fork/DT_ForkItemData.json",
+    "inventory": "DataTable/Inventory/DT_ItemConfig.json",
+    "capital_inventory": "DataTable/Inventory/DT_CapitalItemConfig.json",
+    "appearances": "DataTable/Character/Appearance/DT_AppearanceData.json",
+    "illustrations": "DataTable/Gacha/GachaIllustrate.json",
+    "mystery_box_pools": "DataTable/GashaponLottery/DT_GashaponLotteryGlobal.json",
+    "vehicle_inventory": "DataTable/Vehicle/DT_VehicleItemData.json",
+    "localization": "Localization/en/game.json",
+}
+REWARD_MAPPING_FILES = ("arcs.json", "characters.json", "items.json")
+RANK_BY_QUALITY = {
+    "EItemQuality::ITEM_QUALITY_ORANGE": "S",
+    "EItemQuality::ITEM_QUALITY_PURPLE": "A",
+    "EItemQuality::ITEM_QUALITY_BLUE": "B",
+}
+
+
+class MappingUpdateError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class AssetSource:
+    tables: dict[str, dict[str, Any]]
+    source: str
+    source_ref: str | None
+    sha256: str
+    file_sha256: dict[str, str]
+
+
+@dataclass(frozen=True)
+class MappingUpdateResult:
+    mappings: dict[str, dict[str, dict[str, Any]]]
+    report: dict[str, Any]
+
+    @property
+    def change_count(self) -> int:
+        changes = self.report["changes"]
+        return changes["additions"] + changes["updates"] + changes["deletions"]
+
+
+def load_assets(*, assets_root: Path | None = None, source_ref: str = DEFAULT_SOURCE_REF) -> AssetSource:
+    tables: dict[str, dict[str, Any]] = {}
+    file_hashes: dict[str, str] = {}
+    combined = hashlib.sha256()
+
+    for label, relative_path in ASSET_TABLES.items():
+        if assets_root is not None:
+            path = assets_root / Path(relative_path)
+            try:
+                raw = path.read_bytes()
+            except OSError as exc:
+                raise MappingUpdateError(f"cannot read NTE_Assets table {path}: {exc}") from exc
+        else:
+            url = f"https://raw.githubusercontent.com/Waifus-Grace/NTE_Assets/{source_ref}/{relative_path}"
+            request = urllib.request.Request(url, headers={"User-Agent": "nte-history-exporter-mapping-update"})
+            try:
+                with urllib.request.urlopen(request, timeout=60) as response:
+                    raw = response.read()
+            except OSError as exc:
+                raise MappingUpdateError(f"cannot download NTE_Assets table {relative_path}: {exc}") from exc
+
+        try:
+            document = json.loads(raw.decode("utf-8-sig"))
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise MappingUpdateError(f"invalid NTE_Assets JSON in {relative_path}: {exc}") from exc
+        tables[label] = (
+            _extract_localization(document, relative_path)
+            if label == "localization"
+            else _extract_rows(document, relative_path)
+        )
+        digest = hashlib.sha256(raw).hexdigest()
+        file_hashes[relative_path] = digest
+        combined.update(relative_path.encode("utf-8"))
+        combined.update(b"\0")
+        combined.update(raw)
+
+    source = str(assets_root.resolve()) if assets_root is not None else NTE_ASSETS_REPOSITORY
+    return AssetSource(
+        tables=tables,
+        source=source,
+        source_ref=None if assets_root is not None else source_ref,
+        sha256=combined.hexdigest(),
+        file_sha256=file_hashes,
+    )
+
+
+def load_current_mappings(directory: Path) -> dict[str, dict[str, dict[str, Any]]]:
+    mappings = {}
+    for filename in REWARD_MAPPING_FILES:
+        path = directory / filename
+        try:
+            value = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise MappingUpdateError(f"cannot read {path}: {exc}") from exc
+        if not isinstance(value, dict):
+            raise MappingUpdateError(f"{path} must contain an object")
+        mappings[filename] = value
+    validate_mappings(mappings)
+    return mappings
+
+
+def build_mapping_update(
+    current: dict[str, dict[str, dict[str, Any]]],
+    assets: AssetSource,
+) -> MappingUpdateResult:
+    validate_mappings(current)
+    translations = _build_translation_index(assets.tables["localization"])
+    output = {
+        "arcs.json": _build_primary_mapping(assets.tables["arcs"], "arc", translations),
+        "characters.json": _build_primary_mapping(assets.tables["characters"], "character", translations),
+        "items.json": _build_item_mapping(assets.tables, translations),
+    }
+    validate_mappings(output)
+
+    additions: dict[str, list[str]] = {}
+    updates: dict[str, list[str]] = {}
+    deletions: dict[str, list[str]] = {}
+    for filename in REWARD_MAPPING_FILES:
+        old = current[filename]
+        new = output[filename]
+        additions[filename] = sorted(new.keys() - old.keys(), key=str.casefold)
+        deletions[filename] = sorted(old.keys() - new.keys(), key=str.casefold)
+        updates[filename] = sorted(
+            (item_id for item_id in old.keys() & new.keys() if old[item_id] != new[item_id]),
+            key=str.casefold,
+        )
+
+    report = {
+        "schema_version": 2,
+        "source": assets.source,
+        "source_ref": assets.source_ref,
+        "source_sha256": assets.sha256,
+        "source_file_sha256": assets.file_sha256,
+        "safety": {
+            "authoritative_snapshot": True,
+            "deletions_allowed": True,
+            "pool_mappings_touched": False,
+            "uid_inputs_touched": False,
+        },
+        "changes": {
+            "additions": sum(map(len, additions.values())),
+            "updates": sum(map(len, updates.values())),
+            "deletions": sum(map(len, deletions.values())),
+        },
+        "additions_by_file": additions,
+        "updates_by_file": updates,
+        "deletions_by_file": deletions,
+        "output_counts": {filename: len(entries) for filename, entries in output.items()},
+    }
+    return MappingUpdateResult(output, report)
+
+
+def write_update(result: MappingUpdateResult, directory: Path) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    for filename in REWARD_MAPPING_FILES:
+        _atomic_write(directory / filename, _dump_mapping(result.mappings[filename]))
+    report_text = json.dumps(result.report, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
+    _atomic_write(directory / "mapping-update-report.json", report_text)
+
+
+def apply_update(result: MappingUpdateResult, directory: Path) -> None:
+    """Replace all reward maps as one transaction, rolling back on failure."""
+    directory.mkdir(parents=True, exist_ok=True)
+    originals: dict[Path, bytes | None] = {}
+    written: list[Path] = []
+    for filename in REWARD_MAPPING_FILES:
+        path = directory / filename
+        originals[path] = path.read_bytes() if path.exists() else None
+    try:
+        for filename in REWARD_MAPPING_FILES:
+            path = directory / filename
+            _atomic_write(path, _dump_mapping(result.mappings[filename]))
+            written.append(path)
+    except OSError:
+        for path in reversed(written):
+            original = originals[path]
+            if original is None:
+                path.unlink(missing_ok=True)
+            else:
+                _atomic_write_bytes(path, original)
+        raise
+
+
+def validate_mappings(mappings: dict[str, dict[str, dict[str, Any]]]) -> None:
+    if set(mappings) != set(REWARD_MAPPING_FILES):
+        raise MappingUpdateError(f"mapping set must be exactly {REWARD_MAPPING_FILES}")
+    locations: dict[str, str] = {}
+    folded: dict[str, str] = {}
+    for filename in REWARD_MAPPING_FILES:
+        entries = mappings[filename]
+        if not isinstance(entries, dict):
+            raise MappingUpdateError(f"{filename} must contain an object")
+        for item_id, meta in entries.items():
+            if not isinstance(item_id, str) or not item_id:
+                raise MappingUpdateError(f"{filename} contains an invalid item ID")
+            if item_id in locations:
+                raise MappingUpdateError(f"item ID {item_id} appears in {locations[item_id]} and {filename}")
+            locations[item_id] = filename
+            case_key = item_id.casefold()
+            if case_key in folded and folded[case_key] != item_id:
+                raise MappingUpdateError(f"case-insensitive duplicate IDs: {folded[case_key]} and {item_id}")
+            folded[case_key] = item_id
+            if not isinstance(meta, dict):
+                raise MappingUpdateError(f"{filename}:{item_id} must be an object")
+            name = meta.get("name")
+            if not isinstance(name, str) or not name.strip():
+                raise MappingUpdateError(f"{filename}:{item_id}.name must be a non-empty string")
+            rank = meta.get("rank")
+            if filename == "arcs.json" and rank not in {"S", "A", "B"}:
+                raise MappingUpdateError(f"{filename}:{item_id}.rank must be S, A, or B")
+            if filename == "characters.json" and rank not in {"S", "A"}:
+                raise MappingUpdateError(f"{filename}:{item_id}.rank must be S or A")
+            if filename == "items.json":
+                if meta.get("type") not in {"item", "cosmetic"}:
+                    raise MappingUpdateError(f"{filename}:{item_id}.type must be item or cosmetic")
+                if rank not in {"S", "A", "B"}:
+                    raise MappingUpdateError(f"{filename}:{item_id}.rank must be S, A, or B")
+
+
+def _extract_rows(document: Any, relative_path: str) -> dict[str, Any]:
+    candidates = document if isinstance(document, list) else [document]
+    for candidate in candidates:
+        if not isinstance(candidate, dict):
+            continue
+        rows = candidate.get("Rows")
+        if isinstance(rows, dict):
+            return rows
+        if all(isinstance(key, str) for key in candidate) and candidate:
+            return candidate
+    raise MappingUpdateError(f"NTE_Assets table {relative_path} does not contain a Rows object")
+
+
+def _extract_localization(document: Any, relative_path: str) -> dict[str, Any]:
+    if not isinstance(document, dict) or not all(isinstance(value, dict) for value in document.values()):
+        raise MappingUpdateError(f"NTE_Assets localization {relative_path} must contain namespace objects")
+    return document
+
+
+def _build_translation_index(localization: dict[str, Any]) -> dict[str, list[tuple[str, str]]]:
+    result: dict[str, list[tuple[str, str]]] = {}
+    for namespace, entries in localization.items():
+        for key, value in entries.items():
+            if not isinstance(key, str) or not isinstance(value, str):
+                raise MappingUpdateError(f"invalid English localization entry {namespace}:{key}")
+            result.setdefault(key, []).append((namespace, value))
+    return result
+
+
+def _build_primary_mapping(
+    rows: dict[str, Any],
+    kind: str,
+    translations: dict[str, list[tuple[str, str]]],
+) -> dict[str, dict[str, Any]]:
+    result = {}
+    for item_id in sorted(rows, key=lambda value: (value.casefold(), value)):
+        meta = _normalise_row(item_id, rows[item_id], translations)
+        if kind == "character" and meta["rank"] not in {"S", "A"}:
+            raise MappingUpdateError(f"unexpected character quality for {item_id}")
+        result[item_id] = {"name": meta["name"], "rank": meta["rank"]}
+    return result
+
+
+def _build_item_mapping(
+    tables: dict[str, dict[str, Any]],
+    translations: dict[str, list[tuple[str, str]]],
+) -> dict[str, dict[str, Any]]:
+    inventory = _casefold_index(tables["inventory"], "inventory")
+    capital = _casefold_index(tables["capital_inventory"], "capital inventory")
+    appearances = _casefold_index(tables["appearances"], "appearances")
+    vehicle_inventory = _casefold_index(tables["vehicle_inventory"], "vehicle inventory")
+    result: dict[str, dict[str, Any]] = {}
+    candidate_ids = _reward_candidate_ids(tables)
+    for candidate_id in candidate_ids:
+        folded = candidate_id.casefold()
+        if candidate_id.isdigit() or folded.startswith("fork_") or folded.startswith("characterawaken_"):
+            continue
+        sources = (inventory, capital, appearances, vehicle_inventory)
+        match = next((source.get(folded) for source in sources if folded in source), None)
+        if match is None:
+            raise MappingUpdateError(f"reward {candidate_id} is missing from item tables")
+        canonical_id, row = match
+        if folded.startswith("fashion_glide_") and folded in appearances:
+            canonical_id = appearances[folded][0]
+        meta = _normalise_row(canonical_id, row, translations)
+        result[canonical_id] = {
+            "type": _item_mapping_type(canonical_id),
+            "name": meta["name"],
+            "rank": meta["rank"],
+        }
+    return dict(sorted(result.items(), key=lambda pair: (pair[0].casefold(), pair[0])))
+
+
+def _reward_candidate_ids(tables: dict[str, dict[str, Any]]) -> list[str]:
+    """Return rewards used by ordinary Gacha and every Mystery Box rotation."""
+    candidates: dict[str, str] = {
+        item_id.casefold(): item_id for item_id in tables["illustrations"]
+    }
+    for pool_id, pool in tables["mystery_box_pools"].items():
+        if not isinstance(pool, dict):
+            raise MappingUpdateError(f"Mystery Box pool {pool_id} must be an object")
+        gifts = pool.get("GiftList")
+        if not isinstance(gifts, list):
+            raise MappingUpdateError(f"Mystery Box pool {pool_id} has no GiftList")
+        for index, gift in enumerate(gifts):
+            if not isinstance(gift, dict):
+                raise MappingUpdateError(
+                    f"Mystery Box pool {pool_id} gift {index} must be an object"
+                )
+            item_id = gift.get("ItemID")
+            if not isinstance(item_id, str) or not item_id:
+                raise MappingUpdateError(
+                    f"Mystery Box pool {pool_id} gift {index} has no ItemID"
+                )
+            candidates.setdefault(item_id.casefold(), item_id)
+    return sorted(candidates.values(), key=lambda value: (value.casefold(), value))
+
+
+def _item_mapping_type(item_id: str) -> str:
+    folded = item_id.casefold()
+    cosmetic_prefixes = ("fashion_", "frame_", "bussinesscard_")
+    return "cosmetic" if folded.startswith(cosmetic_prefixes) else "item"
+
+
+def _casefold_index(rows: dict[str, Any], label: str) -> dict[str, tuple[str, Any]]:
+    result: dict[str, tuple[str, Any]] = {}
+    for item_id, row in rows.items():
+        folded = item_id.casefold()
+        if folded in result and result[folded][0] != item_id:
+            raise MappingUpdateError(f"{label} has case-insensitive duplicate IDs: {result[folded][0]} and {item_id}")
+        result[folded] = (item_id, row)
+    return result
+
+
+def _normalise_row(
+    item_id: str,
+    row: Any,
+    translations: dict[str, list[tuple[str, str]]],
+) -> dict[str, str]:
+    if not isinstance(row, dict):
+        raise MappingUpdateError(f"NTE_Assets row {item_id} must be an object")
+    item_name = row.get("ItemName") or row.get("Name")
+    if not isinstance(item_name, dict):
+        raise MappingUpdateError(f"NTE_Assets row {item_id} has no ItemName")
+    name = _translate_name(item_id, item_name, translations)
+    quality = row.get("ItemQuality") or row.get("Quality")
+    rank = RANK_BY_QUALITY.get(quality)
+    if rank is None:
+        raise MappingUpdateError(f"NTE_Assets row {item_id} has unsupported quality {quality!r}")
+    return {"name": name.strip(), "rank": rank}
+
+
+def _translate_name(
+    item_id: str,
+    string_reference: dict[str, Any],
+    translations: dict[str, list[tuple[str, str]]],
+) -> str:
+    key = string_reference.get("Key")
+    if not isinstance(key, str) or not key:
+        raise MappingUpdateError(f"NTE_Assets row {item_id} has no localization key")
+    table_id = string_reference.get("TableId")
+    expected_namespace = None
+    if isinstance(table_id, str) and table_id:
+        expected_namespace = table_id.rsplit("/", 1)[-1].split(".", 1)[0]
+    matches = translations.get(key, [])
+    preferred = [value for namespace, value in matches if namespace == expected_namespace]
+    if len(preferred) == 1:
+        name = preferred[0]
+    elif len(matches) == 1:
+        name = matches[0][1]
+    elif not matches:
+        raise MappingUpdateError(f"English localization is missing {item_id} key {key}")
+    else:
+        namespaces = ", ".join(namespace for namespace, _value in matches)
+        raise MappingUpdateError(f"English localization key {key} for {item_id} is ambiguous: {namespaces}")
+    if not name.strip():
+        raise MappingUpdateError(f"English localization key {key} for {item_id} is empty")
+    return name
+
+
+def _dump_mapping(mapping: dict[str, dict[str, Any]]) -> str:
+    lines = ["{"]
+    entries = list(mapping.items())
+    for index, (item_id, meta) in enumerate(entries):
+        comma = "," if index < len(entries) - 1 else ""
+        key = json.dumps(item_id, ensure_ascii=False)
+        value = json.dumps(meta, ensure_ascii=False, separators=(", ", ": "))
+        lines.append(f"  {key}: {value}{comma}")
+    lines.append("}")
+    return "\n".join(lines) + "\n"
+
+
+def _atomic_write(path: Path, text: str) -> None:
+    _atomic_write_bytes(path, text.encode("utf-8"))
+
+
+def _atomic_write_bytes(path: Path, content: bytes) -> None:
+    with tempfile.NamedTemporaryFile(
+        mode="wb", dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False
+    ) as handle:
+        handle.write(content)
+        temporary = Path(handle.name)
+    try:
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)

src/nte_history_exporter/pool_mappings.py

@@ -8,6 +8,7 @@ POOL_MAPPING_FILES = {
     "permanent": "permanent_board.json",
     "limited_character": "limited_character_board.json",
     "arc_miracle_box": "arc_miracle_box.json",
+    "mystery_box": "mystery_box.json",
 }
 
 

tests/__init__.py

@@ -0,0 +1 @@
+"""Test package for the NTE history exporter."""

tests/fixtures/README.md

@@ -0,0 +1,38 @@
+# Synthetic network fixture
+
+`synthetic_history_network.json` is the committed end-to-end packet fixture for
+the exporter test suite. It contains UDP payloads for Permanent Board and Arc
+history requests/responses, plus isolated protocol edge cases.
+
+The fixture is safe to publish:
+
+- endpoints use the RFC 5737 documentation ranges `192.0.2.0/24` and
+  `198.51.100.0/24`;
+- packet and history timestamps are deterministic synthetic values;
+- it contains no game user UID, account/session data, cookies, tokens, or raw
+  capture metadata;
+- the test suite checks every payload with the UID extractor before accepting
+  it.
+
+Tests load the fixture through helpers in `tests/support.py`. The `replay`
+scenario is fed through `LiveHistorySession`; `protocol_sample` packets cover
+points gifts, chase rewards, batched pages, and bit-packed responses.
+
+Do not replace this file with a real `.pcap`, `.flows`, or exported account
+history. Add new cases by constructing the smallest relevant payload, replacing
+all timestamps and endpoints, and extending the privacy assertions.
+
+`synthetic_capture_diagnostics.json` is a smaller replay transcript for failure
+paths. It deliberately contains a short response and a marker-free response so
+the debug sidecar's reason codes and privacy contract can be tested without a
+real capture. Repeated-byte payloads use `payload_byte` plus `payload_length` to
+keep the fixture readable.
+
+## Synthetic NTE_Assets fixture
+
+`nte_assets/` mirrors only the six table paths and English localization file
+consumed by the mapping updater. Its rows, IDs, and names are deliberately
+synthetic and contain no account or personal data. The fixture exercises
+authoritative additions, metadata updates, deletions, translation lookup,
+illustration filtering, and canonical ID casing without depending on the
+network or copying a live game-data snapshot into the repository.

tests/fixtures/nte_assets/DataTable/Character/Appearance/DT_AppearanceData.json

@@ -0,0 +1 @@
+[{"Rows":{"Fashion_Glide_2000":{"Name":{"TableId":"/Game/Text/ST_Appearance.ST_Appearance","Key":"glider_2000","LocalizedString":"Wrong Embedded Appearance"},"Quality":"EItemQuality::ITEM_QUALITY_PURPLE"}}}]

tests/fixtures/nte_assets/DataTable/Character/DT_Character.json

@@ -0,0 +1 @@
+[{"Rows":{"1003":{"ItemName":{"TableId":"/Game/Text/ST_Player.ST_Player","Key":"character_1003","LocalizedString":"Wrong Embedded Character"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"},"1099":{"ItemName":{"TableId":"/Game/Text/ST_Player.ST_Player","Key":"character_1099","LocalizedString":"Wrong Embedded New Character"},"ItemQuality":"EItemQuality::ITEM_QUALITY_PURPLE"}}}]

tests/fixtures/nte_assets/DataTable/Fork/DT_ForkItemData.json

@@ -0,0 +1 @@
+[{"Rows":{"fork_alpha":{"ItemName":{"TableId":"/Game/Text/ST_Fork.ST_Fork","Key":"fork_alpha_name","LocalizedString":"Wrong Embedded Alpha"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"},"fork_new":{"ItemName":{"TableId":"/Game/Text/ST_Fork.ST_Fork","Key":"fork_new_name","LocalizedString":"Wrong Embedded New Arc"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"}}}]

tests/fixtures/nte_assets/DataTable/Gacha/GachaIllustrate.json

@@ -0,0 +1 @@
+[{"Rows":{"1003":{},"1099":{},"fork_alpha":{},"fork_new":{},"DIceNormal":{},"Dice_ticket_03":{},"Fashion_glide_2000":{},"Characterawaken_1003":{}}}]

tests/fixtures/nte_assets/DataTable/GashaponLottery/DT_GashaponLotteryGlobal.json

@@ -0,0 +1 @@
+[{"Rows":{}}]

tests/fixtures/nte_assets/DataTable/Inventory/DT_CapitalItemConfig.json

@@ -0,0 +1 @@
+[{"Rows":{}}]

tests/fixtures/nte_assets/DataTable/Inventory/DT_ItemConfig.json

@@ -0,0 +1 @@
+[{"Rows":{"DiceNormal":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"dice_normal","LocalizedString":"Wrong Embedded Dice"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"},"Dice_ticket_03":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"ticket_03","LocalizedString":"Wrong Embedded Ticket"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"},"Fashion_glide_2000":{"ItemName":{"TableId":"/Game/Text/ST_Appearance.ST_Appearance","Key":"glider_2000","LocalizedString":"Wrong Embedded Glider"},"ItemQuality":"EItemQuality::ITEM_QUALITY_PURPLE"},"UnusedItem":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"unused_item","LocalizedString":"Wrong Embedded Unused"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"}}}]

tests/fixtures/nte_assets/DataTable/Vehicle/DT_VehicleItemData.json

@@ -0,0 +1 @@
+[{"Rows":{}}]

tests/fixtures/nte_assets/Localization/en/game.json

@@ -0,0 +1,18 @@
+{
+  "ST_Player": {
+    "character_1003": "Old Character Renamed",
+    "character_1099": "New Character"
+  },
+  "ST_Fork": {
+    "fork_alpha_name": "Alpha Arc",
+    "fork_new_name": "New Arc"
+  },
+  "ST_Item": {
+    "dice_normal": "Fabricated Dice",
+    "ticket_03": "New Ticket",
+    "unused_item": "Not A Pull Reward"
+  },
+  "ST_Appearance": {
+    "glider_2000": "Inventory Glider"
+  }
+}

tests/fixtures/synthetic_capture_diagnostics.json

@@ -0,0 +1,60 @@
+{
+  "schema_version": 1,
+  "description": "Minimal synthetic replay for capture diagnostic reason codes.",
+  "privacy": {
+    "synthetic": true,
+    "contains_user_uid": false,
+    "contains_raw_account_session": false
+  },
+  "local_ip": "192.0.2.10",
+  "expected": {
+    "packets_seen": 3,
+    "history_requests_recognized": 1,
+    "response_candidates_rejected": 2,
+    "event_counts": {
+      "HISTORY_REQUEST_RECOGNIZED": 1,
+      "NO_HISTORY_MARKER": 1,
+      "RESPONSE_TOO_SHORT": 1
+    },
+    "reason_counts": {
+      "NO_HISTORY_MARKER": 1,
+      "RESPONSE_TOO_SHORT": 1
+    },
+    "pending_response_candidates": 2,
+    "pending_candidate_lengths": [80, 220]
+  },
+  "packets": [
+    {
+      "label": "permanent-page-1-request",
+      "timestamp": 1893456000.0,
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 50000,
+      "dst_port": 40000,
+      "protocol": "udp",
+      "payload_hex": "00000000000000000000000000000000000000000000000000000000000000040000007c100000000400000000"
+    },
+    {
+      "label": "matching-response-too-short",
+      "timestamp": 1893456000.1,
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 40000,
+      "dst_port": 50000,
+      "protocol": "udp",
+      "payload_byte": "00",
+      "payload_length": 80
+    },
+    {
+      "label": "matching-response-without-history-marker",
+      "timestamp": 1893456000.2,
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 40000,
+      "dst_port": 50000,
+      "protocol": "udp",
+      "payload_byte": "00",
+      "payload_length": 220
+    }
+  ]
+}

tests/fixtures/synthetic_history_network.json

@@ -0,0 +1,322 @@
+{
+  "type": "nte-synthetic-network-fixture",
+  "schema_version": 1,
+  "description": "Synthetic UDP history transcript for tests. Contains no real account or network identity data.",
+  "privacy": {
+    "synthetic": true,
+    "contains_user_uid": false,
+    "contains_raw_account_session": false,
+    "addresses": "RFC 5737 documentation ranges",
+    "timestamps": "deterministic synthetic values"
+  },
+  "local_ip": "192.0.2.10",
+  "expected": {
+    "permanent_records": 25,
+    "arc_records": 25,
+    "permanent_first_uid": "bdf0395ad3fb21f89b62ca92aaef7e36",
+    "arc_first_uid": "221b6accc3de16a145293b8804761c7c"
+  },
+  "packets": [
+    {
+      "type": "packet",
+      "label": "permanent-page-1-request",
+      "scenario": "replay",
+      "timestamp": 1.0,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "00000000000000000000000000000000000000000000000000000000000000040000007c100000000400000000"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-1-response",
+      "scenario": "replay",
+      "timestamp": 1.01,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-2-request",
+      "scenario": "replay",
+      "timestamp": 1.02,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "00000000000000000000000000000000000000000000000000000000000000080000007c100000000400000000"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-2-response",
+      "scenario": "replay",
+      "timestamp": 1.03,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-3-request",
+      "scenario": "replay",
+      "timestamp": 1.04,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000c0000007c100000000400000000"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-3-response",
+      "scenario": "replay",
+      "timestamp": 1.05,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-4-request",
+      "scenario": "replay",
+      "timestamp": 1.06,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "00000000000000000000000000000000000000000000000000000000000000100000007c100000000400000000"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-4-response",
+      "scenario": "replay",
+      "timestamp": 1.07,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-5-request",
+      "scenario": "replay",
+      "timestamp": 1.08,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "00000000000000000000000000000000000000000000000000000000000000140000007c100000000400000000"
+    },
+    {
+      "type": "packet",
+      "label": "permanent-page-5-response",
+      "scenario": "replay",
+      "timestamp": 1.09,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b2300c805000014000000100000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c800440000000c85c99141bdbdb17d3995dd49bdb195010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-1-request",
+      "scenario": "replay",
+      "timestamp": 1.1,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "0000000000000000000000000000000000000000000000000c080000000200000000"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-1-response",
+      "scenario": "replay",
+      "timestamp": 1.11,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-2-request",
+      "scenario": "replay",
+      "timestamp": 1.12,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "0000000000000000000000000000000000000000000000000c080000000400000000"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-2-response",
+      "scenario": "replay",
+      "timestamp": 1.1300000000000001,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-3-request",
+      "scenario": "replay",
+      "timestamp": 1.1400000000000001,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "0000000000000000000000000000000000000000000000000c080000000600000000"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-3-response",
+      "scenario": "replay",
+      "timestamp": 1.1500000000000001,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-4-request",
+      "scenario": "replay",
+      "timestamp": 1.1600000000000001,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "0000000000000000000000000000000000000000000000000c080000000800000000"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-4-response",
+      "scenario": "replay",
+      "timestamp": 1.1700000000000002,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-5-request",
+      "scenario": "replay",
+      "timestamp": 1.1800000000000002,
+      "protocol": "udp",
+      "src_ip": "192.0.2.10",
+      "dst_ip": "198.51.100.20",
+      "src_port": 49152,
+      "dst_port": 30031,
+      "payload_hex": "0000000000000000000000000000000000000000000000000c080000000a00000000"
+    },
+    {
+      "type": "packet",
+      "label": "arc-page-5-response",
+      "scenario": "replay",
+      "timestamp": 1.1900000000000002,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000016000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc51116000000ccdee4d6bedcdedcdee600280000008cdee4d698dee8e8cae4f2be82dc90eadca2ea000000cdd9418bc511"
+    },
+    {
+      "type": "packet",
+      "label": "limited-points-gift-1",
+      "scenario": "protocol_sample",
+      "timestamp": 11.0,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e4050000140000000000000014000000c4c0c8c000000000000400000014000000c4c0c8c0004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "limited-chase-reward",
+      "scenario": "protocol_sample",
+      "timestamp": 12.0,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009806000014000000fcffffff4b00000010a58d957dd1a58dad95d17dc1c4b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c4004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "limited-batched-pages",
+      "scenario": "protocol_sample",
+      "timestamp": 13.0,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e805000014000000140000002c00000098bdc9ad7db9bdb9bdcd0100000000040000002c00000098bdc9ad7db9bdb9bdcd014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230400000014000000c4c0c8c000000000000400000014000000c4c0c8c0004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100000002c00000098bdc9ad7db9bdb9bdcd0100000000040000002c00000098bdc9ad7db9bdb9bdcd014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000002800000098bdc9ad7dd9a5b9950100000000080000003c00000010a58d957dd1a58dad95d17dc1c4002800000098bdc9ad7dd9a5b995014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000003400000098bdc9ad7d91d5cdd189a5b90100000000040000003400000098bdc9ad7d91d5cdd189a5b9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100126675671c6e0ecfcb769070000000c08000000100000003e0800000004000000486c0000001835bdb9bdc1bdb1e531bdd1d195c9e549958dbdc9911185d18501000406000014000000100000003000000098bdc9ad7de585bd9185bd0100000000040000003000000098bdc9ad7de585bd9185bd014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230c00000014000000c4c0d4c80000000000080000003c00000010a58d957dd1a58dad95d17dc1c40014000000c4c0d4c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23140000004800000010a58d957dd1a58dad95d17dc1c8b0d4c00000000000040000003c00000010a58d957dd1a58dad95d17dc1c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b231800000014000000c4c0c0e000000000000400000014000000c4c0c0e0004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100000003c00000098bdc9ad7d41c9bdad85c9e5bdb90100000000040000003c00000098bdc9ad7d41c9bdad85c9e5bdb9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "limited-bitpacked-source",
+      "scenario": "protocol_sample",
+      "timestamp": 14.0,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004c06000014000000180000003c00000098bdc9ad7d41c9bdad85c9e5bdb90100000000080000003c00000010a58d957dd1a58dad95d17dc1c4003c00000098bdc9ad7d41c9bdad85c9e5bdb9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230000000014000000c4c0c4e400000000000400000014000000c4c0c4e4004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000003400000098bdc9ad7d91d5cdd189a5b90100000000080000003c00000010a58d957dd1a58dad95d17dc1c4003400000098bdc9ad7d91d5cdd189a5b9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23100000003c00000098bdc9ad7d85c1c1b1a585b98d950100000000040000003c00000098bdc9ad7d85c1c1b1a585b98d95014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b231000000014000000c4c0cccc00000000000400000014000000c4c0cccc004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23005406000014000000080000004000000098bdc9ad7dddd5a1d585add585b99d0100000000040000004000000098bdc9ad7dddd5a1d585add585b99d014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b230c0000002c00000010a58d95b1a5b5a5d1950100000000040000002c00000010a58d95b1a5b5a5d195014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23040000003400000098bdc9ad7d91d5cdd189a5b90100000000040000003400000098bdc9ad7d91d5cdd189a5b9014c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23180000004800000010a58d957dd1a58dad95d17dc1c8b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23180000004800000010a58d957dd1a58dad95d17dc1c8b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c8004c0000000c85c99141bdbdb17d0da185c9858dd195c9010000f3095f138b23"
+    },
+    {
+      "type": "packet",
+      "label": "limited-bitpacked-response",
+      "scenario": "protocol_sample",
+      "timestamp": 15.0,
+      "protocol": "udp",
+      "src_ip": "198.51.100.20",
+      "dst_ip": "192.0.2.10",
+      "src_port": 30031,
+      "dst_port": 49152,
+      "payload_hex": "15000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080c9000080020000000300008007000000b337b9b52f28b9b7b530b9bc373700000000000100008007000000a2b4b1b22fbab4b1b532ba2f98188007000000b337b9b52f28b9b7b530b9bc37378009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b62710400000080020000801898981c000000008000000080020000801898981c8009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271840000008006000000b337b9b52fb2ba393ab1343700000000000100008007000000a2b4b1b22fbab4b1b532ba2f98188006000000b337b9b52fb2ba393ab134378009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271040200008007000000b337b9b5af303838b6b430b7b13200000000800000008007000000b337b9b5af303838b6b430b7b1328009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b62710402000080020000801898991900000000800000008002000080189899198009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b62710480ca000080020000000100000008000000b337b9b5afbb3ab4bab0b5ba30b73300000000800000000008000000b337b9b5afbb3ab4bab0b5ba30b7338009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271840100008005000000a2b4b132b6b4b634ba3200000000800000008005000000a2b4b132b6b4b634ba328009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271840000008006000000b337b9b52fb2ba393ab1343700000000800000008006000000b337b9b52fb2ba393ab134378009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271040300000009000000a2b4b1b22fbab4b1b532ba2f181996191800000000800000008007000000a2b4b1b22fbab4b1b532ba2f18198009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b6271040300000009000000a2b4b1b22fbab4b1b532ba2f181996191800000000800000008007000000a2b4b1b22fbab4b1b532ba2f18198009000080a1303932a8b737b6af21b430b9b031ba32390000603ee16b627104"
+    }
+  ]
+}
+

tests/support.py

@@ -0,0 +1,123 @@
+import csv
+import json
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import Mock, patch
+from tempfile import TemporaryDirectory
+
+ROOT = Path(__file__).resolve().parents[1]
+SRC = ROOT / "src"
+FIXTURES = ROOT / "tests" / "fixtures"
+if str(SRC) not in sys.path:
+    sys.path.insert(0, str(SRC))
+
+from nte_history_exporter import __version__
+from nte_history_exporter.decoder.boundary import annotate_groups, make_uid
+from nte_history_exporter.constants import LIMITED_CHARACTER_MARKER, MARKER
+from nte_history_exporter.decoder.boundary import select_continuous_run_from_page_1
+from nte_history_exporter.decoder.protocol import decode_response_records, history_request_kind
+from nte_history_exporter.constants import POOL_META
+from nte_history_exporter.mappings import ARC_META, CHARACTERS, ITEMS, REWARDS_BY_ID
+from nte_history_exporter.decoder.protocol import decode_reward_key, infer_reward_type
+from nte_history_exporter.decoder.user_uid import extract_user_uid
+from nte_history_exporter.decoder.server_region import extract_server_id
+from nte_history_exporter.export.csv_export import write_csv
+from nte_history_exporter.decoder.arc import (
+    arc_request_page,
+    build_arc_rows_from_pairs,
+    decode_arc_key,
+    decode_arc_timestamp,
+    is_arc_history_request,
+    make_arc_uid,
+    parse_arc_response,
+)
+from nte_history_exporter.live_capture.session import LiveHistorySession, UdpPacket
+from nte_history_exporter.live_capture.libpcap import (
+    DLT_EN10MB,
+    DLT_LINUX_SLL,
+    DLT_LINUX_SLL2,
+    DLT_LOOP,
+    DLT_RAW,
+    _extract_ipv4_frame,
+    _load_library,
+    LibpcapUnavailable,
+)
+from nte_history_exporter.live_capture.windows_raw import parse_ipv4_packet
+from nte_history_exporter.live_capture.backends import open_capture_backend
+from nte_history_exporter.export.json_export import build_export_json
+from nte_history_exporter.live_capture.runner import export_paths
+from nte_history_exporter import console
+from nte_history_exporter.pool_mappings import load_pool_mappings, pool_meta_from_mapping
+from nte_history_exporter.update_check import UpdateInfo, check_for_update, is_newer_version
+
+
+def decode_single_record(record_hex):
+    return decode_response_records(bytes(0x50) + bytes.fromhex(record_hex))[0]
+
+
+def load_network_fixture():
+    path = FIXTURES / "synthetic_history_network.json"
+    with path.open(encoding="utf-8") as f:
+        return json.load(f)
+
+
+def load_capture_diagnostics_fixture():
+    path = FIXTURES / "synthetic_capture_diagnostics.json"
+    with path.open(encoding="utf-8") as f:
+        return json.load(f)
+
+
+def diagnostic_fixture_session():
+    fixture = load_capture_diagnostics_fixture()
+    session = LiveHistorySession(fixture["local_ip"])
+    for packet in fixture["packets"]:
+        if "payload_hex" in packet:
+            payload = bytes.fromhex(packet["payload_hex"])
+        else:
+            payload = bytes.fromhex(packet["payload_byte"]) * packet["payload_length"]
+        session.process_packet(
+            UdpPacket(
+                timestamp=packet["timestamp"],
+                src_ip=packet["src_ip"],
+                dst_ip=packet["dst_ip"],
+                src_port=packet["src_port"],
+                dst_port=packet["dst_port"],
+                payload=payload,
+                protocol=packet["protocol"],
+            )
+        )
+    return session
+
+
+def fixture_packets(scenario=None):
+    fixture = load_network_fixture()
+    packets = fixture["packets"]
+    if scenario is not None:
+        packets = [packet for packet in packets if packet["scenario"] == scenario]
+    return [
+        UdpPacket(
+            timestamp=packet["timestamp"],
+            src_ip=packet["src_ip"],
+            dst_ip=packet["dst_ip"],
+            src_port=packet["src_port"],
+            dst_port=packet["dst_port"],
+            payload=bytes.fromhex(packet["payload_hex"]),
+            protocol=packet["protocol"],
+        )
+        for packet in packets
+    ]
+
+
+def fixture_payload(label):
+    fixture = load_network_fixture()
+    packet = next(packet for packet in fixture["packets"] if packet["label"] == label)
+    return bytes.fromhex(packet["payload_hex"])
+
+
+def fixture_session():
+    fixture = load_network_fixture()
+    session = LiveHistorySession(fixture["local_ip"])
+    for packet in fixture_packets("replay"):
+        session.process_packet(packet)
+    return session

tests/test_arc_decoding.py

@@ -0,0 +1,120 @@
+from tests.support import *  # noqa: F401,F403
+
+
+class ArcDecodingTests(unittest.TestCase):
+    def test_arc_key_timestamp_and_uid_match_fixture(self):
+        fixture = load_network_fixture()
+        row = fixture_session().build_rows("arc_miracle_box")[0]
+        self.assertEqual(decode_arc_key(bytes.fromhex(row["reward_key_hex"])), "fork_nonos")
+        _ticks, _unix, decoded = decode_arc_timestamp(bytes.fromhex(row["timestamp_raw_hex"]))
+        self.assertEqual(decoded, "2030-01-02 00:00:00")
+        self.assertEqual(row["uid"], fixture["expected"]["arc_first_uid"])
+
+    def test_arc_response_parser_matches_fixture_first_page(self):
+        decoded = parse_arc_response(fixture_payload("arc-page-1-response"))
+        self.assertEqual(len(decoded), 5)
+        self.assertEqual([row["reward_id"] for row in decoded], ["fork_nonos"] * 5)
+        self.assertEqual(decoded[0]["reward_type"], "arc")
+        self.assertEqual(decode_arc_key(bytes.fromhex(decoded[0]["reward_key_hex"])), "fork_nonos")
+
+    def test_arc_response_parser_rejects_invalid_timestamp_noise(self):
+        response = bytearray(0x4C)
+        response += (10).to_bytes(4, "little")
+        response += bytes.fromhex("ccdee4d6be")
+        response += (8).to_bytes(4, "little")
+        response += b"garb"
+        response += (0xFFFFFFFFFFFFFFFF).to_bytes(8, "little")
+
+        self.assertEqual(parse_arc_response(bytes(response)), [])
+
+    def test_arc_partial_timestamp_group_is_exported_without_warning(self):
+        decoded = fixture_session().build_rows("arc_miracle_box")
+        exported = [row for row in decoded if row["export_record"] is True]
+
+        # The oldest group is a 10-pull split by stopping at page 5 (5 of 10 rows).
+        # Its captured prefix is ordinal-stable, so every row is exported with a
+        # stable UID and no warning.
+        self.assertEqual(len(decoded), 25)
+        self.assertEqual(len(exported), 25)
+        self.assertTrue(all(row["uid"] for row in decoded))
+        self.assertTrue(all(row["uid_status"] == "stable" for row in decoded))
+
+    def test_arc_incomplete_prefix_keeps_stable_uids(self):
+        pairs = fixture_session().pairs_for_kind("arc_miracle_box")
+        full_pairs = pairs[:2]
+        trunc_pairs = pairs[:1]
+        ts = parse_arc_response(fixture_payload("arc-page-1-response"))[0]["timestamp_raw_hex"]
+        full = [r for r in build_arc_rows_from_pairs(full_pairs) if r["timestamp_raw_hex"] == ts]
+        trunc = [r for r in build_arc_rows_from_pairs(trunc_pairs) if r["timestamp_raw_hex"] == ts]
+        self.assertTrue(trunc)
+        self.assertEqual([r["uid"] for r in trunc], [r["uid"] for r in full[: len(trunc)]])
+
+    def test_arc_row_builder_accepts_live_pairs_with_kind(self):
+        response = fixture_payload("arc-page-1-response")
+        decoded = build_arc_rows_from_pairs([(1, 2, 1, 1.0, 2, 1.1, response, "arc_miracle_box")])
+        self.assertEqual(len(decoded), 5)
+        self.assertEqual(decoded[0]["reward_id"], "fork_nonos")
+
+    def test_arc_export_is_shared_pity(self):
+        rows = fixture_session().build_rows("arc_miracle_box")
+        export = build_export_json(rows, [])
+        self.assertEqual(export["banner"]["id"], "Arc_MiracleBox")
+        self.assertIs(export["banner"]["shared_pity"], True)
+
+    def test_group_detection_counts_only_dice_records_but_uid_ordinals_keep_all_rows(self):
+        rows = [
+            {
+                "page": 1,
+                "timestamp_raw_hex": "aa",
+                "timestamp_decoded": "2026-01-01 00:00:00",
+                "result_type": "dice",
+                "dice": 1,
+                "reward_key_hex": "k1",
+                "quantity": 1,
+            },
+            {
+                "page": 1,
+                "timestamp_raw_hex": "aa",
+                "timestamp_decoded": "2026-01-01 00:00:00",
+                "result_type": "points_gift",
+                "dice": 0,
+                "reward_key_hex": "k2",
+                "quantity": 1,
+            },
+            {
+                "page": 1,
+                "timestamp_raw_hex": "aa",
+                "timestamp_decoded": "2026-01-01 00:00:00",
+                "result_type": "chase_reward",
+                "dice": -4,
+                "reward_key_hex": "k3",
+                "quantity": 30,
+            },
+            {
+                "page": 1,
+                "timestamp_raw_hex": "aa",
+                "timestamp_decoded": "2026-01-01 00:00:00",
+                "result_type": "dice",
+                "dice": 2,
+                "reward_key_hex": "k4",
+                "quantity": 1,
+            },
+            {
+                "page": 1,
+                "timestamp_raw_hex": "bb",
+                "timestamp_decoded": "2026-01-01 00:01:00",
+                "result_type": "dice",
+                "dice": 3,
+                "reward_key_hex": "k5",
+                "quantity": 1,
+            },
+        ]
+
+        annotated = annotate_groups(rows)
+
+        # Ordinals cover every row in the group, but the dice-only count drives
+        # timestamp_group_size_seen (2 dice in the 4-record group).
+        self.assertEqual([row["timestamp_group_ordinal"] for row in annotated[:4]], [0, 1, 2, 3])
+        self.assertEqual({row["timestamp_group_size_seen"] for row in annotated[:4]}, {2})
+        self.assertEqual({row["timestamp_group_record_size_seen"] for row in annotated[:4]}, {4})
+        self.assertTrue(all(row["export_record"] for row in annotated))

tests/test_capture_backends.py

@@ -0,0 +1,92 @@
+from tests.support import *  # noqa: F401,F403
+from nte_history_exporter.live_capture.libpcap import _extract_ipv4_frame, _load_library
+
+
+class CaptureBackendTests(unittest.TestCase):
+    def test_windows_auto_prefers_npcap(self):
+        capture = Mock(device=r"\Device\NPF_test")
+        with (
+            patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"),
+            patch(
+                "nte_history_exporter.live_capture.backends.open_libpcap_capture",
+                return_value=capture,
+            ),
+            patch("nte_history_exporter.live_capture.backends.RawSocketCapture") as raw_capture,
+        ):
+            selected = open_capture_backend("192.0.2.1")
+
+        self.assertIs(selected, capture)
+        self.assertEqual(selected.name, "npcap")
+        self.assertEqual(selected.fallback_reason, "")
+        raw_capture.assert_not_called()
+
+    def test_windows_auto_falls_back_to_raw_socket_without_npcap(self):
+        fallback = Mock()
+        with (
+            patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"),
+            patch(
+                "nte_history_exporter.live_capture.backends.open_libpcap_capture",
+                side_effect=LibpcapUnavailable("Npcap could not be loaded"),
+            ),
+            patch(
+                "nte_history_exporter.live_capture.backends.RawSocketCapture",
+                return_value=fallback,
+            ) as raw_capture,
+        ):
+            selected = open_capture_backend("192.0.2.1")
+
+        self.assertIs(selected, fallback)
+        raw_capture.assert_called_once_with(
+            "192.0.2.1",
+            fallback_reason="Npcap could not be loaded",
+        )
+
+    def test_explicit_libpcap_does_not_fall_back(self):
+        with (
+            patch("nte_history_exporter.live_capture.backends.sys.platform", "win32"),
+            patch(
+                "nte_history_exporter.live_capture.backends.open_libpcap_capture",
+                side_effect=LibpcapUnavailable("Npcap could not be loaded"),
+            ),
+            patch("nte_history_exporter.live_capture.backends.RawSocketCapture") as raw_capture,
+            self.assertRaises(LibpcapUnavailable),
+        ):
+            open_capture_backend("192.0.2.1", "libpcap")
+
+        raw_capture.assert_not_called()
+
+    def test_windows_libpcap_load_prefers_system_then_path(self):
+        system_dir = Path("C:/Windows/System32/Npcap")
+        loaded = Mock()
+        with (
+            patch("nte_history_exporter.live_capture.libpcap.sys.platform", "win32"),
+            patch("nte_history_exporter.live_capture.libpcap._windows_npcap_directory", return_value=system_dir),
+            patch("nte_history_exporter.live_capture.libpcap.Path.is_dir", return_value=True),
+            patch("nte_history_exporter.live_capture.libpcap.os.add_dll_directory", return_value=Mock(), create=True),
+            patch(
+                "nte_history_exporter.live_capture.libpcap.ctypes.CDLL",
+                side_effect=[OSError("system missing"), loaded],
+            ) as cdll,
+        ):
+            self.assertIs(_load_library(), loaded)
+
+        self.assertEqual(
+            [call.args[0] for call in cdll.call_args_list],
+            [
+                str(system_dir / "wpcap.dll"),
+                "wpcap.dll",
+            ],
+        )
+
+    def test_libpcap_link_layers_extract_ipv4_packets(self):
+        ip_packet = bytes.fromhex("4500001c0000000040110000c0000201c6336402") + bytes(8)
+        ethernet = bytes(12) + bytes.fromhex("0800") + ip_packet
+        loop = (2).to_bytes(4, sys.byteorder) + ip_packet
+        linux_sll = bytes(14) + bytes.fromhex("0800") + ip_packet
+        linux_sll2 = bytes.fromhex("0800") + bytes(18) + ip_packet
+
+        self.assertEqual(_extract_ipv4_frame(ethernet, DLT_EN10MB), ip_packet)
+        self.assertEqual(_extract_ipv4_frame(ip_packet, DLT_RAW), ip_packet)
+        self.assertEqual(_extract_ipv4_frame(loop, DLT_LOOP), ip_packet)
+        self.assertEqual(_extract_ipv4_frame(linux_sll, DLT_LINUX_SLL), ip_packet)
+        self.assertEqual(_extract_ipv4_frame(linux_sll2, DLT_LINUX_SLL2), ip_packet)

tests/test_capture_diagnostics.py

@@ -0,0 +1,108 @@
+from tests.support import *  # noqa: F401,F403
+
+from nte_history_exporter.live_capture.diagnostics import (
+    new_diagnostics_path,
+    write_capture_diagnostics,
+)
+
+
+class CaptureDiagnosticsTests(unittest.TestCase):
+    def test_successful_network_replay_reports_capture_pipeline_counts(self):
+        report = fixture_session().diagnostic_report()
+
+        self.assertEqual(
+            report["counters"],
+            {
+                "history_requests_recognized": 10,
+                "history_responses_decoded": 10,
+                "packets_seen": 20,
+                "pages_matched": 10,
+                "udp_packets_seen": 20,
+            },
+        )
+        self.assertEqual(report["reason_counts"], {})
+        self.assertEqual(report["pending_requests"], [])
+
+    def test_synthetic_replay_reports_actionable_rejection_reasons(self):
+        fixture = load_capture_diagnostics_fixture()
+        report = diagnostic_fixture_session().diagnostic_report()
+        expected = fixture["expected"]
+
+        self.assertEqual(report["format"], "nte-capture-diagnostics")
+        self.assertEqual(report["format_version"], 1)
+        self.assertEqual(report["counters"]["packets_seen"], expected["packets_seen"])
+        self.assertEqual(
+            report["counters"]["history_requests_recognized"],
+            expected["history_requests_recognized"],
+        )
+        self.assertEqual(
+            report["counters"]["response_candidates_rejected"],
+            expected["response_candidates_rejected"],
+        )
+        self.assertEqual(report["event_counts"], expected["event_counts"])
+        self.assertEqual(report["reason_counts"], expected["reason_counts"])
+        self.assertEqual(report["pending_requests"][0]["response_candidates"], 2)
+        self.assertEqual(
+            report["pending_requests"][0]["response_candidate_lengths"],
+            expected["pending_candidate_lengths"],
+        )
+
+    def test_diagnostic_fixture_contains_only_synthetic_network_identity(self):
+        fixture = load_capture_diagnostics_fixture()
+        self.assertTrue(fixture["privacy"]["synthetic"])
+        self.assertFalse(fixture["privacy"]["contains_user_uid"])
+        self.assertFalse(fixture["privacy"]["contains_raw_account_session"])
+
+        allowed_ips = {"192.0.2.10", "198.51.100.20"}
+        for packet in fixture["packets"]:
+            with self.subTest(label=packet["label"]):
+                self.assertIn(packet["src_ip"], allowed_ips)
+                self.assertIn(packet["dst_ip"], allowed_ips)
+                if "payload_hex" in packet:
+                    payload = bytes.fromhex(packet["payload_hex"])
+                else:
+                    payload = bytes.fromhex(packet["payload_byte"]) * packet["payload_length"]
+                self.assertIsNone(extract_user_uid(payload))
+
+    def test_diagnostic_report_excludes_capture_identity_and_payload_data(self):
+        report = diagnostic_fixture_session().diagnostic_report()
+        serialized = json.dumps(report)
+        forbidden_keys = {
+            "src_ip",
+            "dst_ip",
+            "src_port",
+            "dst_port",
+            "timestamp",
+            "payload",
+            "payload_hex",
+            "user_uid",
+        }
+
+        def assert_safe(value):
+            if isinstance(value, dict):
+                self.assertTrue(forbidden_keys.isdisjoint(value))
+                for child in value.values():
+                    assert_safe(child)
+            elif isinstance(value, list):
+                for child in value:
+                    assert_safe(child)
+
+        assert_safe(report)
+        self.assertNotIn("192.0.2.10", serialized)
+        self.assertNotIn("198.51.100.20", serialized)
+        self.assertNotIn("1893456000", serialized)
+
+    def test_diagnostics_are_debug_sidecar_not_public_export_data(self):
+        session = diagnostic_fixture_session()
+        export = build_export_json([], [])
+        self.assertNotIn("diagnostics", export)
+        self.assertNotIn("capture_diagnostics", export)
+
+        with TemporaryDirectory() as temp_dir:
+            diagnostics_path = new_diagnostics_path(temp_dir)
+            write_capture_diagnostics(diagnostics_path, session.diagnostic_report())
+            written = json.loads(diagnostics_path.read_text(encoding="utf-8"))
+
+        self.assertEqual(written, session.diagnostic_report())
+        self.assertRegex(diagnostics_path.name, r"^Capture_\d{8}_\d{6}\.diagnostics\.json$")
+        self.assertNotIn("user", diagnostics_path.name.casefold())

tests/test_export_contract.py

@@ -0,0 +1,179 @@
+from tests.support import *  # noqa: F401,F403
+from nte_history_exporter import __version__
+
+
+class ExportContractTests(unittest.TestCase):
+    def test_sanitized_export_omits_raw_packet_fields(self):
+        annotated = annotate_groups(fixture_session().build_rows("permanent"))
+        export = build_export_json(annotated, [])
+
+        self.assertEqual(export["format"], "nte-history-export")
+        self.assertIn("exporter", export)
+        self.assertNotIn("user_uid", export)
+        self.assertNotIn("server_id", export)
+        self.assertNotIn("account_region", export)
+        self.assertNotIn("record_hex", export["records"][0])
+        self.assertNotIn("request_msg", export["records"][0])
+        self.assertNotIn("response_msg", export["records"][0])
+
+    def test_export_includes_user_uid_when_provided(self):
+        annotated = annotate_groups(fixture_session().build_rows("permanent"))
+        export = build_export_json(
+            annotated,
+            [],
+            capture_source="npcap",
+            user_uid="123456789",
+        )
+
+        self.assertEqual(list(export).index("user_uid"), list(export).index("records") - 1)
+        self.assertEqual(export["capture_source"], "npcap")
+        self.assertEqual(export["user_uid"], "123456789")
+
+    def test_export_includes_server_id_and_mapped_account_region(self):
+        annotated = annotate_groups(fixture_session().build_rows("permanent"))
+        export = build_export_json(annotated, [], server_id="23003")
+
+        self.assertEqual(export["server_id"], "23003")
+        self.assertEqual(export["account_region"], "EU")
+
+    def test_export_preserves_unknown_server_without_guessing_region(self):
+        annotated = annotate_groups(fixture_session().build_rows("permanent"))
+        export = build_export_json(annotated, [], server_id="23999")
+
+        self.assertEqual(export["server_id"], "23999")
+        self.assertNotIn("account_region", export)
+
+    def test_extracts_server_id_from_valid_initial_tcp_response(self):
+        payload = bytearray(204)
+        payload[0:4] = (200).to_bytes(4, "little")
+        payload[4:8] = (20).to_bytes(4, "little")
+        payload[96:100] = (23003).to_bytes(4, "little")
+        address = b"198.51.100.20"
+        payload[132:136] = len(address).to_bytes(4, "little")
+        payload[136 : 136 + len(address)] = address
+
+        self.assertEqual(extract_server_id(bytes(payload)), "23003")
+
+    def test_rejects_server_id_at_offset_without_valid_message_structure(self):
+        payload = bytearray(204)
+        payload[96:100] = (23003).to_bytes(4, "little")
+
+        self.assertIsNone(extract_server_id(bytes(payload)))
+
+    def test_live_session_detects_server_only_on_inbound_tcp(self):
+        payload = bytearray(204)
+        payload[0:4] = (200).to_bytes(4, "little")
+        payload[4:8] = (20).to_bytes(4, "little")
+        payload[96:100] = (23004).to_bytes(4, "little")
+        address = b"198.51.100.20"
+        payload[132:136] = len(address).to_bytes(4, "little")
+        payload[136 : 136 + len(address)] = address
+        session = LiveHistorySession("192.0.2.10")
+
+        session.process_packet(
+            UdpPacket(
+                1.0,
+                "198.51.100.20",
+                "192.0.2.10",
+                30000,
+                40000,
+                bytes(payload),
+                protocol="tcp",
+            )
+        )
+
+        self.assertEqual(session.server_id, "23004")
+
+    @patch("builtins.input", side_effect=["9", "3"])
+    def test_server_prompt_retries_then_returns_selected_server_id(self, _input):
+        self.assertEqual(console.prompt_server_id(), "23003")
+
+    @patch("builtins.input", return_value="")
+    def test_server_prompt_can_be_skipped(self, _input):
+        self.assertIsNone(console.prompt_server_id())
+
+    @patch("nte_history_exporter.console.wait_for_keypress")
+    def test_wait_for_close_waits_for_a_second_keypress(self, wait_for_keypress):
+        console.wait_for_close()
+
+        wait_for_keypress.assert_called_once_with()
+
+    def test_debug_csv_includes_exporter_version(self):
+        with TemporaryDirectory() as tmp:
+            path = Path(tmp) / "debug.csv"
+            write_csv(path, [{"uid": "abc123"}])
+
+            with path.open(newline="", encoding="utf-8") as f:
+                rows = list(csv.DictReader(f))
+
+        self.assertEqual(rows[0]["exporter_version"], __version__)
+        self.assertEqual(rows[0]["uid"], "abc123")
+
+    def test_export_paths_include_user_uid_banner_and_timestamp(self):
+        _csv_path, json_path = export_paths("limited_character", "218216016349")
+
+        self.assertRegex(
+            json_path.name,
+            r"^218216016349_Limited_\d{8}_\d{6}(?:_\d+)?\.json$",
+        )
+
+    def test_extracts_user_uid_from_record_context(self):
+        payload = (
+            b"\x00" * 24
+            + (218216016349).to_bytes(8, "little")
+            + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00"
+        )
+
+        self.assertEqual(extract_user_uid(payload), "218216016349")
+
+    def test_extracts_user_uid_from_private_spawn_record_context(self):
+        payload = (
+            b"\x88\x00\x00\x00\x10\x00\x00\x00"
+            + (218216016349).to_bytes(8, "little")
+            + b"\x08\x00\x0c\x00\x07\x00\x08\x00\x08\x00\x00\x00"
+            + b"\x00\x00\x00\x01\x08\x00\x00\x00\x04\x00\x04\x00"
+            + b"\x04\x00\x00\x00\x16\x00\x00\x00PrivateSpawnInfoRecord\x00"
+        )
+
+        self.assertEqual(extract_user_uid(payload), "218216016349")
+
+    def test_does_not_extract_user_uid_from_wrong_record_offset(self):
+        payload = (
+            b"\x00" * 28
+            + (218216016349).to_bytes(8, "little")
+            + b"\x00\x00\x00\x00TagOthers\x00"
+        )
+
+        self.assertIsNone(extract_user_uid(payload))
+
+    def test_does_not_extract_old_eight_digit_false_positive_as_user_uid(self):
+        payload = (
+            b"WholeVehicleData\x00\x00\x00\x00\x00o<\x00\x00\x05\x00\x00\x00"
+            b"\x0b\x00\x00\x00Vehicle015\x00\x0b\x00\x00\x00buyvehicle\x00"
+            b"\x09\x00\x00\x0015363624\x00\x06\x00\x00\x00"
+        )
+
+        self.assertIsNone(extract_user_uid(payload))
+
+    def test_ipv4_parser_extracts_tcp_payload_for_user_uid_detection(self):
+        payload = (
+            (218216016349).to_bytes(8, "little")
+            + b"\x00\x00\x00\x00\x09\x00\x00\x00TagOthers\x00"
+        )
+        tcp_header = bytearray(20)
+        tcp_header[0:2] = (40000).to_bytes(2, "big")
+        tcp_header[2:4] = (30000).to_bytes(2, "big")
+        tcp_header[12] = 5 << 4
+        total_len = 20 + len(tcp_header) + len(payload)
+        ip_header = bytearray(20)
+        ip_header[0] = 0x45
+        ip_header[2:4] = total_len.to_bytes(2, "big")
+        ip_header[9] = 6
+        ip_header[12:16] = bytes([192, 0, 2, 1])
+        ip_header[16:20] = bytes([198, 51, 100, 2])
+
+        packet = parse_ipv4_packet(bytes(ip_header) + bytes(tcp_header) + payload)
+
+        self.assertIsNotNone(packet)
+        self.assertEqual(packet.protocol, "tcp")
+        self.assertEqual(packet.payload, payload)

tests/test_live_session.py

@@ -0,0 +1,214 @@
+from tests.support import *  # noqa: F401,F403
+
+
+class LiveSessionTests(unittest.TestCase):
+    def test_live_session_pairs_request_and_response(self):
+        session = LiveHistorySession("192.168.0.10")
+
+        request = bytearray(45)
+        request[31:35] = (4).to_bytes(4, "little")
+        request[35:39] = (4220).to_bytes(4, "little")
+        request[40:44] = (4).to_bytes(4, "little")
+
+        response = bytearray(220)
+        response[0x50:0x50 + len(MARKER)] = MARKER
+        response[0x50 + len(MARKER):0x50 + len(MARKER) + 8] = (
+            2556647947780680000
+        ).to_bytes(8, "little")
+
+        self.assertFalse(
+            session.process_packet(
+                UdpPacket(
+                    timestamp=1.0,
+                    src_ip="192.168.0.10",
+                    dst_ip="203.0.113.5",
+                    src_port=50000,
+                    dst_port=40000,
+                    payload=bytes(request),
+                )
+            )
+        )
+        self.assertTrue(
+            session.process_packet(
+                UdpPacket(
+                    timestamp=1.2,
+                    src_ip="203.0.113.5",
+                    dst_ip="192.168.0.10",
+                    src_port=40000,
+                    dst_port=50000,
+                    payload=bytes(response),
+                )
+            )
+        )
+        self.assertEqual(len(session.pairs), 1)
+        self.assertEqual(session.last_page_seen, 1)
+
+    def test_live_session_pairs_pipelined_and_batched_pages(self):
+        session = LiveHistorySession("192.168.0.10")
+
+        def request_packet(page, timestamp):
+            request = bytearray(45)
+            request[31:35] = (page * 4).to_bytes(4, "little")
+            request[35:39] = (4220).to_bytes(4, "little")
+            request[40:44] = (4).to_bytes(4, "little")
+            return UdpPacket(
+                timestamp=timestamp,
+                src_ip="192.168.0.10",
+                dst_ip="203.0.113.5",
+                src_port=50000,
+                dst_port=40000,
+                payload=bytes(request),
+            )
+
+        timestamp = (2556647947780680000).to_bytes(8, "little")
+
+        def response_packet(record_count, packet_timestamp):
+            response = bytearray(0x50)
+            for _ in range(record_count):
+                response += bytes(4) + MARKER + timestamp
+            return UdpPacket(
+                timestamp=packet_timestamp,
+                src_ip="203.0.113.5",
+                dst_ip="192.168.0.10",
+                src_port=40000,
+                dst_port=50000,
+                payload=bytes(response),
+            )
+
+        for page in range(1, 9):
+            session.process_packet(request_packet(page, 1.0 + page / 10))
+
+        self.assertTrue(session.process_packet(response_packet(10, 2.0)))
+        self.assertEqual([pair[0] for pair in session.pairs], [1, 2])
+        self.assertEqual([pair[8:10] for pair in session.pairs], [(0, 5), (5, 5)])
+
+        self.assertTrue(session.process_packet(response_packet(10, 2.1)))
+        self.assertEqual([pair[0] for pair in session.pairs], [1, 2, 3, 4])
+        self.assertEqual(session.missing_pages("permanent"), [])
+        rows = session.build_rows("permanent")
+        self.assertEqual(len(rows), 20)
+        self.assertEqual(
+            {page: sum(row["page"] == page for row in rows) for page in range(1, 5)},
+            {1: 5, 2: 5, 3: 5, 4: 5},
+        )
+
+        session.process_packet(request_packet(9, 2.2))
+        self.assertTrue(session.process_packet(response_packet(4, 2.3)))
+        self.assertEqual(session.pairs[-1][0], 9)
+        self.assertEqual(session.pairs[-1][8:10], (0, 4))
+
+    def test_live_session_new_page_one_starts_clean_recovery_cycle(self):
+        session = LiveHistorySession("192.168.0.10")
+
+        def request_packet(page, timestamp):
+            request = bytearray(45)
+            request[31:35] = (page * 4).to_bytes(4, "little")
+            request[35:39] = (4220).to_bytes(4, "little")
+            request[40:44] = (4).to_bytes(4, "little")
+            return UdpPacket(
+                timestamp=timestamp,
+                src_ip="192.168.0.10",
+                dst_ip="203.0.113.5",
+                src_port=50000,
+                dst_port=40000,
+                payload=bytes(request),
+            )
+
+        session.process_packet(request_packet(7, 1.0))
+        session.process_packet(request_packet(8, 1.1))
+        session.process_packet(request_packet(1, 2.0))
+
+        self.assertEqual([request.page for request in session.pending], [1])
+        self.assertEqual(
+            session.missing_page_reason("permanent", 7),
+            "request captured; no matching response page was captured",
+        )
+
+        timestamp = (2556647947780680000).to_bytes(8, "little")
+        response = bytearray(0x50)
+        for _ in range(5):
+            response += bytes(4) + MARKER + timestamp
+        self.assertTrue(
+            session.process_packet(
+                UdpPacket(
+                    timestamp=2.1,
+                    src_ip="203.0.113.5",
+                    dst_ip="192.168.0.10",
+                    src_port=40000,
+                    dst_port=50000,
+                    payload=bytes(response),
+                )
+            )
+        )
+        self.assertEqual(session.pairs[-1][0], 1)
+        self.assertEqual(session.missing_pages("permanent"), [])
+
+    def test_live_session_reports_unrecognized_response_candidate(self):
+        session = LiveHistorySession("192.168.0.10")
+
+        def request_packet(page, timestamp):
+            request = bytearray(45)
+            request[31:35] = (page * 4).to_bytes(4, "little")
+            request[35:39] = (4220).to_bytes(4, "little")
+            request[40:44] = (4).to_bytes(4, "little")
+            return UdpPacket(
+                timestamp=timestamp,
+                src_ip="192.168.0.10",
+                dst_ip="203.0.113.5",
+                src_port=50000,
+                dst_port=40000,
+                payload=bytes(request),
+            )
+
+        session.process_packet(request_packet(1, 1.0))
+        session.process_packet(
+            UdpPacket(
+                timestamp=1.1,
+                src_ip="203.0.113.5",
+                dst_ip="192.168.0.10",
+                src_port=40000,
+                dst_port=50000,
+                payload=bytes(220),
+            )
+        )
+        session.process_packet(request_packet(2, 1.2))
+
+        self.assertEqual(
+            session.missing_page_reason("permanent", 1),
+            "1 matching inbound UDP packet(s) captured but not recognized as history response (lengths: 220)",
+        )
+
+    def test_live_session_ignores_non_history_udp_packets(self):
+        session = LiveHistorySession("192.168.0.10")
+
+        request = bytearray(45)
+        request[31:35] = (4).to_bytes(4, "little")
+        request[35:39] = (4220).to_bytes(4, "little")
+        request[40:44] = (4).to_bytes(4, "little")
+
+        noise = b"not-a-history-response" * 20
+
+        session.process_packet(
+            UdpPacket(
+                timestamp=1.0,
+                src_ip="192.168.0.10",
+                dst_ip="203.0.113.5",
+                src_port=50000,
+                dst_port=40000,
+                payload=bytes(request),
+            )
+        )
+        self.assertFalse(
+            session.process_packet(
+                UdpPacket(
+                    timestamp=1.1,
+                    src_ip="203.0.113.5",
+                    dst_ip="192.168.0.10",
+                    src_port=40000,
+                    dst_port=50000,
+                    payload=noise,
+                )
+            )
+        )
+        self.assertEqual(len(session.pairs), 0)
+

tests/test_mapping_update.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+import json
+import unittest
+from copy import deepcopy
+from dataclasses import replace
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+from nte_history_exporter.mapping_update import (
+    MappingUpdateError,
+    apply_update,
+    build_mapping_update,
+    load_assets,
+    load_current_mappings,
+    write_update,
+)
+from tests.support import FIXTURES
+
+
+SAMPLE_ASSETS = FIXTURES / "nte_assets"
+
+
+def sample_current():
+    return {
+        "arcs.json": {
+            "fork_alpha": {"name": "Stale Alpha Name", "rank": "B"},
+            "fork_retired": {"name": "Retired Arc", "rank": "A"},
+        },
+        "characters.json": {
+            "1003": {"name": "Stale Character Name", "rank": "S"},
+            "1001": {"name": "Retired Character", "rank": "A"},
+        },
+        "items.json": {
+            "DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"},
+            "OldTicket": {"type": "item", "name": "Retired Ticket", "rank": "B"},
+        },
+    }
+
+
+def build_sample_update():
+    current = sample_current()
+    return current, build_mapping_update(current, load_assets(assets_root=SAMPLE_ASSETS))
+
+
+class MappingUpdateTests(unittest.TestCase):
+    def test_mapping_update_is_an_authoritative_snapshot(self):
+        current, result = build_sample_update()
+
+        self.assertEqual(result.mappings["characters.json"]["1003"], {"name": "Old Character Renamed", "rank": "S"})
+        self.assertEqual(result.mappings["characters.json"]["1099"], {"name": "New Character", "rank": "A"})
+        self.assertNotIn("1001", result.mappings["characters.json"])
+        self.assertNotIn("fork_retired", result.mappings["arcs.json"])
+        self.assertNotIn("OldTicket", result.mappings["items.json"])
+        self.assertNotEqual(result.mappings["characters.json"]["1003"], current["characters.json"]["1003"])
+        self.assertEqual(result.report["changes"], {"additions": 4, "updates": 2, "deletions": 3})
+        self.assertIs(result.report["safety"]["authoritative_snapshot"], True)
+        self.assertIs(result.report["safety"]["deletions_allowed"], True)
+        self.assertIs(result.report["safety"]["uid_inputs_touched"], False)
+        self.assertIs(result.report["safety"]["pool_mappings_touched"], False)
+
+    def test_items_use_illustrations_as_filter_and_asset_tables_as_authority(self):
+        _current, result = build_sample_update()
+
+        self.assertEqual(
+            result.mappings["items.json"]["Fashion_Glide_2000"],
+            {"type": "cosmetic", "name": "Inventory Glider", "rank": "A"},
+        )
+        self.assertIn("DiceNormal", result.mappings["items.json"])
+        self.assertNotIn("DIceNormal", result.mappings["items.json"])
+        self.assertNotIn("UnusedItem", result.mappings["items.json"])
+        self.assertFalse(any(key.startswith("Characterawaken_") for key in result.mappings["items.json"]))
+
+    def test_future_mystery_box_pool_is_discovered_without_hard_coded_event_id(self):
+        assets = load_assets(assets_root=SAMPLE_ASSETS)
+        tables = deepcopy(assets.tables)
+        tables["mystery_box_pools"]["MangHe_wowzers"] = {
+            "GiftList": [
+                {"ItemID": "WowTicket"},
+                {"ItemID": "Frame_Wowzers"},
+                {"ItemID": "VehicleWow"},
+            ]
+        }
+        tables["inventory"]["WowTicket"] = {
+            "ItemName": {"TableId": "/Game/Text/ST_Item.ST_Item", "Key": "wow_ticket"},
+            "ItemQuality": "EItemQuality::ITEM_QUALITY_PURPLE",
+        }
+        tables["inventory"]["Frame_Wowzers"] = {
+            "ItemName": {"TableId": "/Game/Text/ST_Item.ST_Item", "Key": "wow_frame"},
+            "ItemQuality": "EItemQuality::ITEM_QUALITY_BLUE",
+        }
+        tables["vehicle_inventory"]["vehiclewow"] = {
+            "ItemName": {
+                "TableId": "/Game/Text/ST_VehicleData.ST_VehicleData",
+                "Key": "vehicle_wow",
+            },
+            "ItemQuality": "EItemQuality::ITEM_QUALITY_ORANGE",
+        }
+        tables["localization"]["ST_Item"].update(
+            {"wow_ticket": "Wowzers Ticket", "wow_frame": "Wowzers Frame"}
+        )
+        tables["localization"]["ST_VehicleData"] = {"vehicle_wow": "Wowmobile"}
+
+        result = build_mapping_update(sample_current(), replace(assets, tables=tables))
+
+        self.assertEqual(
+            result.mappings["items.json"]["WowTicket"],
+            {"type": "item", "name": "Wowzers Ticket", "rank": "A"},
+        )
+        self.assertEqual(
+            result.mappings["items.json"]["Frame_Wowzers"],
+            {"type": "cosmetic", "name": "Wowzers Frame", "rank": "B"},
+        )
+        self.assertEqual(
+            result.mappings["items.json"]["vehiclewow"],
+            {"type": "item", "name": "Wowmobile", "rank": "S"},
+        )
+
+    def test_embedded_localized_strings_are_ignored(self):
+        _current, result = build_sample_update()
+
+        self.assertEqual(result.mappings["characters.json"]["1003"]["name"], "Old Character Renamed")
+        self.assertNotEqual(result.mappings["characters.json"]["1003"]["name"], "Wrong Embedded Character")
+
+    def test_mapping_update_rejects_missing_english_translation(self):
+        assets = load_assets(assets_root=SAMPLE_ASSETS)
+        tables = deepcopy(assets.tables)
+        del tables["localization"]["ST_Player"]["character_1003"]
+
+        with self.assertRaisesRegex(MappingUpdateError, "English localization is missing"):
+            build_mapping_update(sample_current(), replace(assets, tables=tables))
+
+    def test_mapping_update_rejects_ambiguous_translation_without_namespace_match(self):
+        assets = load_assets(assets_root=SAMPLE_ASSETS)
+        tables = deepcopy(assets.tables)
+        tables["inventory"]["Fashion_glide_2000"]["ItemName"]["TableId"] = "/Game/Text/Unknown.Unknown"
+        tables["localization"]["OtherNamespace"] = {"glider_2000": "Other Glider"}
+
+        with self.assertRaisesRegex(MappingUpdateError, "is ambiguous"):
+            build_mapping_update(sample_current(), replace(assets, tables=tables))
+
+    def test_local_asset_source_has_reproducible_provenance(self):
+        first = load_assets(assets_root=SAMPLE_ASSETS)
+        second = load_assets(assets_root=SAMPLE_ASSETS)
+
+        self.assertEqual(first.sha256, second.sha256)
+        self.assertEqual(first.file_sha256, second.file_sha256)
+        self.assertEqual(first.source, str(SAMPLE_ASSETS.resolve()))
+        self.assertIsNone(first.source_ref)
+
+    def test_mapping_update_writes_deterministic_review_artifacts(self):
+        _current, result = build_sample_update()
+        with TemporaryDirectory() as first_tmp, TemporaryDirectory() as second_tmp:
+            first = Path(first_tmp)
+            second = Path(second_tmp)
+            write_update(result, first)
+            write_update(result, second)
+
+            for filename in (*result.mappings, "mapping-update-report.json"):
+                self.assertEqual((first / filename).read_bytes(), (second / filename).read_bytes())
+            report = json.loads((first / "mapping-update-report.json").read_text(encoding="utf-8"))
+            self.assertEqual(report["changes"]["deletions"], 3)
+
+    def test_mapping_update_does_not_mutate_inputs(self):
+        current = sample_current()
+        original = deepcopy(current)
+
+        build_mapping_update(current, load_assets(assets_root=SAMPLE_ASSETS))
+
+        self.assertEqual(current, original)
+
+    def test_mapping_update_apply_touches_only_reward_mapping_files(self):
+        _current, result = build_sample_update()
+        with TemporaryDirectory() as tmp:
+            directory = Path(tmp)
+            for filename, mapping in sample_current().items():
+                (directory / filename).write_text(json.dumps(mapping), encoding="utf-8")
+            protected = directory / "permanent_board.json"
+            protected.write_text('{"protected": true}\n', encoding="utf-8")
+
+            apply_update(result, directory)
+
+            self.assertEqual(load_current_mappings(directory), result.mappings)
+            self.assertEqual(protected.read_text(encoding="utf-8"), '{"protected": true}\n')
+            self.assertFalse((directory / "mapping-update-report.json").exists())
+
+    def test_mapping_update_apply_rolls_back_partial_write(self):
+        _current, result = build_sample_update()
+        with TemporaryDirectory() as tmp:
+            directory = Path(tmp)
+            originals = {}
+            for filename, mapping in sample_current().items():
+                destination = directory / filename
+                destination.write_text(json.dumps(mapping), encoding="utf-8")
+                originals[filename] = destination.read_bytes()
+
+            from nte_history_exporter import mapping_update
+
+            real_write = mapping_update._atomic_write
+            calls = 0
+
+            def fail_second_write(path, text):
+                nonlocal calls
+                calls += 1
+                if calls == 2:
+                    raise OSError("synthetic write failure")
+                return real_write(path, text)
+
+            with (
+                patch("nte_history_exporter.mapping_update._atomic_write", side_effect=fail_second_write),
+                self.assertRaisesRegex(OSError, "synthetic write failure"),
+            ):
+                apply_update(result, directory)
+
+            for filename, original in originals.items():
+                self.assertEqual((directory / filename).read_bytes(), original)
+
+    def test_mapping_update_rejects_unresolved_illustrated_rewards(self):
+        assets = load_assets(assets_root=SAMPLE_ASSETS)
+        tables = deepcopy(assets.tables)
+        tables["illustrations"]["MissingReward"] = {}
+
+        with self.assertRaisesRegex(MappingUpdateError, "MissingReward"):
+            build_mapping_update(sample_current(), replace(assets, tables=tables))
+
+    def test_load_assets_rejects_missing_tables(self):
+        with TemporaryDirectory() as tmp, self.assertRaisesRegex(MappingUpdateError, "cannot read"):
+            load_assets(assets_root=Path(tmp))

tests/test_mappings_and_updates.py

@@ -0,0 +1,122 @@
+from tests.support import *  # noqa: F401,F403
+
+
+class MappingAndUpdateTests(unittest.TestCase):
+    def test_pool_mapping_json_files_have_uniform_shape(self):
+        required_top_level = {"pool_key", "game", "system", "banner", "request", "response"}
+        for pool_key, mapping in load_pool_mappings().items():
+            with self.subTest(pool_key=pool_key):
+                self.assertEqual(set(required_top_level) - set(mapping), set())
+                self.assertEqual(mapping["pool_key"], pool_key)
+                self.assertIn("id", mapping["system"])
+                self.assertIn("name", mapping["system"])
+                self.assertIn("id", mapping["banner"])
+                self.assertIn("name", mapping["banner"])
+                self.assertIn("shared_pity", mapping["banner"])
+                self.assertIn("family", mapping["request"])
+                self.assertIn("length", mapping["request"])
+                self.assertIn("constant", mapping["request"])
+                self.assertIn("cursor_step", mapping["request"])
+
+    def test_pool_mapping_json_matches_runtime_pool_meta(self):
+        for pool_key, mapping in load_pool_mappings().items():
+            with self.subTest(pool_key=pool_key):
+                self.assertEqual(pool_meta_from_mapping(mapping), POOL_META[pool_key])
+
+    def test_update_version_comparison_handles_release_tags(self):
+        self.assertTrue(is_newer_version("v0.1.7", "0.1.6"))
+        self.assertTrue(is_newer_version("0.2.0", "0.1.6"))
+        self.assertTrue(is_newer_version("v0.1.10", "0.1.9"))
+        self.assertFalse(is_newer_version("v0.1.6", "0.1.6"))
+        self.assertFalse(is_newer_version("v0.1.5", "0.1.6"))
+        self.assertFalse(is_newer_version("latest", "0.1.6"))
+
+    def test_update_check_reports_newer_github_release(self):
+        latest = {
+            "tag_name": "v0.1.7",
+            "html_url": "https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.7",
+        }
+        with patch("nte_history_exporter.update_check.fetch_latest_release", return_value=latest):
+            update = check_for_update("0.1.6", timeout=0.1)
+
+        self.assertEqual(
+            update,
+            UpdateInfo(
+                current_version="0.1.6",
+                latest_version="v0.1.7",
+                release_url="https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.7",
+            ),
+        )
+
+    def test_update_check_ignores_prerelease(self):
+        latest = {
+            "tag_name": "v0.1.8-dev-branch.123",
+            "html_url": "https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.8-dev-branch.123",
+            "prerelease": True,
+        }
+        with patch("nte_history_exporter.update_check.fetch_latest_release", return_value=latest):
+            self.assertIsNone(check_for_update("0.1.7", timeout=0.1))
+
+    def test_update_check_is_quiet_when_unavailable_or_current(self):
+        with patch("nte_history_exporter.update_check.fetch_latest_release", side_effect=OSError("offline")):
+            self.assertIsNone(check_for_update("0.1.6", timeout=0.1))
+
+        with patch("nte_history_exporter.update_check.fetch_latest_release", return_value={"tag_name": "v0.1.6"}):
+            self.assertIsNone(check_for_update("0.1.6", timeout=0.1))
+
+    def test_reward_mapping_files_have_expected_shape(self):
+        self.assertTrue(ARC_META)
+        for arc_id, meta in ARC_META.items():
+            with self.subTest(arc_id=arc_id):
+                self.assertTrue(arc_id.startswith("fork_"))
+                self.assertIn("name", meta)
+                self.assertIn(meta.get("rank"), ("S", "A", "B"))
+
+        self.assertTrue(CHARACTERS)
+        for character_id, info in CHARACTERS.items():
+            with self.subTest(character_id=character_id):
+                self.assertTrue(character_id.isdigit())
+                self.assertIn("name", info)
+                self.assertIn(info.get("rank"), ("S", "A"))
+
+        self.assertTrue(ITEMS)
+        for item_id, info in ITEMS.items():
+            with self.subTest(item_id=item_id):
+                self.assertIn(info.get("type"), ("item", "cosmetic"))
+                self.assertIn("name", info)
+
+    def test_rewards_by_id_merges_all_mapping_files(self):
+        for reward_id in (*ARC_META, *CHARACTERS, *ITEMS):
+            with self.subTest(reward_id=reward_id):
+                reward = REWARDS_BY_ID[reward_id]
+                self.assertEqual(reward["id"], reward_id)
+                self.assertIn(reward["type"], ("arc", "character", "item", "cosmetic"))
+
+    def test_decode_reward_key_round_trips_observed_keys(self):
+        observed = {
+            "98bdc9ad7dd9a5b99501": "fork_vine",
+            "98bdc9ad7d41c9bdad85c9e5bdb901": "fork_Prokaryon",
+            "98bdc9ad7ddda1d585add585b99d01": "fork_whuakuang",
+            "98bdc9ad7dddd5a1d585add585b99d01": "fork_wuhuakuang",
+            "10a58d9539bdc9b585b101": "DiceNormal",
+            "10a58d957dd1a58dad95d17dc1c400": "Dice_ticket_01",
+            "10a58d957dd1a58dad95d17dc1c800": "Dice_ticket_02",
+            "10a58d95b1a5b5a5d19501": "Dicelimite",
+            "1885cda1a5bdb97d1db1a591957dc5c0c4c000": "Fashion_Glide_1010",
+            "1885cda1a5bdb97dd995a1a58db1957dc5c0c4c07c59c1c0e000": "Fashion_vehicle_1010_V008",
+            "c4c0cccc00": "1033",
+            "c4c0dcc000": "1070",
+            "c4c0dcc0": "1070",
+            "c4c0c8c4": "1021",
+        }
+        for key_hex, expected_id in observed.items():
+            with self.subTest(key_hex=key_hex):
+                self.assertEqual(decode_reward_key(bytes.fromhex(key_hex)), expected_id)
+
+    def test_infer_reward_type_for_unmapped_ids(self):
+        self.assertEqual(infer_reward_type("fork_newarc"), "arc")
+        self.assertEqual(infer_reward_type("1099"), "character")
+        self.assertEqual(infer_reward_type("Fashion_hat_2000"), "cosmetic")
+        self.assertEqual(infer_reward_type("Dice_ticket_03"), "item")
+        self.assertEqual(infer_reward_type(""), "")
+

tests/test_mystery_box_decoding.py

@@ -0,0 +1,116 @@
+from tests.support import *  # noqa: F401,F403
+
+from datetime import datetime, timezone
+
+from nte_history_exporter.constants import MYSTERY_BOX_MARKER
+from nte_history_exporter.decoder.mystery_box import (
+    build_mystery_box_rows_from_pairs,
+    is_mystery_box_history_request,
+    mystery_box_request_page,
+    parse_mystery_box_response,
+)
+
+
+def mystery_box_request(page: int) -> bytes:
+    request = bytearray(54)
+    request[26:30] = (2060).to_bytes(4, "little")
+    request[31:35] = (page * 2).to_bytes(4, "little")
+    request[35:39] = (2110).to_bytes(4, "little")
+    request[40:44] = (2).to_bytes(4, "little")
+    request[44:48] = (2060).to_bytes(4, "little")
+    request[49:53] = (10).to_bytes(4, "little")
+    request[53] = 6
+    return bytes(request)
+
+
+def mystery_box_response(records: list[tuple[str, int, datetime]]) -> bytes:
+    body = bytearray()
+    for reward_id, quantity, timestamp in records:
+        encoded_id = reward_id.encode("utf-8") + b"\0"
+        ticks = int(timestamp.timestamp() * 10_000_000) + 621_355_968_000_000_000
+        body += len(encoded_id).to_bytes(4, "little")
+        body += encoded_id
+        body += quantity.to_bytes(4, "little")
+        body += b"\x01"
+        body += ticks.to_bytes(8, "little")
+    return (
+        bytes(50)
+        + MYSTERY_BOX_MARKER
+        + b"\0"
+        + bytes(4)
+        + len(body).to_bytes(4, "little")
+        + len(records).to_bytes(4, "little")
+        + body
+        + b"\x03"
+    )
+
+
+class MysteryBoxDecodingTests(unittest.TestCase):
+    def test_recognizes_request_and_page_cursor(self):
+        request = mystery_box_request(6)
+        self.assertTrue(is_mystery_box_history_request(request))
+        self.assertEqual(mystery_box_request_page(request), 6)
+
+    def test_decodes_exact_quantity_and_timestamp(self):
+        timestamp = datetime(2026, 7, 8, 19, 9, 33, tzinfo=timezone.utc)
+        rows = parse_mystery_box_response(
+            mystery_box_response(
+                [
+                    ("vehicle039", 1, timestamp),
+                    ("SpecialGift_ticket", 3, timestamp),
+                    ("gold", 100_000, timestamp),
+                ]
+            )
+        )
+        self.assertEqual(len(rows), 3)
+        self.assertEqual(rows[0]["reward_id"], "vehicle039")
+        self.assertEqual(rows[0]["reward_name"], "Draco")
+        self.assertEqual(rows[0]["timestamp_decoded"], "2026-07-08 19:09:33")
+        self.assertEqual(rows[1]["quantity"], 3)
+        self.assertEqual(rows[1]["result_type"], "single_pull")
+        self.assertEqual(rows[2]["reward_name"], "Beetle Coin")
+        self.assertEqual(rows[2]["reward_rank"], "B")
+
+    def test_live_session_accepts_partial_final_page(self):
+        session = LiveHistorySession("192.168.0.10")
+        request = mystery_box_request(3)
+        timestamp = datetime(2026, 7, 8, 19, 9, 33, tzinfo=timezone.utc)
+        response = mystery_box_response(
+            [("Fons", 100_000, timestamp), ("gold", 100_000, timestamp)]
+        )
+        self.assertFalse(
+            session.process_packet(
+                UdpPacket(1.0, "192.168.0.10", "203.0.113.5", 50000, 40000, request)
+            )
+        )
+        self.assertTrue(
+            session.process_packet(
+                UdpPacket(1.1, "203.0.113.5", "192.168.0.10", 40000, 50000, response)
+            )
+        )
+        self.assertEqual(session.kinds_seen(), ["mystery_box"])
+        self.assertEqual(session.pairs[0][0], 3)
+        self.assertEqual(session.pairs[0][8:10], (0, 2))
+
+    def test_rows_and_export_use_non_shared_mystery_box_banner(self):
+        timestamp = datetime(2026, 7, 8, 19, 9, 33, tzinfo=timezone.utc)
+        response = mystery_box_response([("Fons", 100_000, timestamp)])
+        rows = build_mystery_box_rows_from_pairs(
+            [(1, 2, 1, 1.0, 2, 1.1, response, "mystery_box", 0, 1)]
+        )
+        export = build_export_json(rows, [])
+        self.assertEqual(export["banner"]["id"], "Gashapon_MysteryBox")
+        self.assertEqual(export["banner"]["name"], "Mystery Box")
+        self.assertIs(export["banner"]["shared_pity"], False)
+        self.assertEqual(export["records"][0]["result_type"], "single_pull")
+        self.assertEqual(export["records"][0]["quantity"], 100_000)
+        self.assertNotIn("roll_result", export["records"][0])
+        self.assertEqual(rows[0]["timestamp_group_size_seen"], 1)
+        self.assertEqual(rows[0]["uid_status"], "stable")
+
+    def test_export_path_uses_mystery_box_prefix(self):
+        _csv_path, json_path = export_paths("mystery_box", "218216016349")
+        self.assertRegex(
+            json_path.name,
+            r"^218216016349_MysteryBox_\d{8}_\d{6}(?:_\d+)?\.json$",
+        )

tests/test_network_fixture.py

@@ -0,0 +1,74 @@
+from tests.support import *  # noqa: F401,F403
+
+
+class SyntheticNetworkFixtureTests(unittest.TestCase):
+    def test_fixture_contains_only_synthetic_network_identity(self):
+        fixture = load_network_fixture()
+
+        self.assertTrue(fixture["privacy"]["synthetic"])
+        self.assertFalse(fixture["privacy"]["contains_user_uid"])
+        self.assertFalse(fixture["privacy"]["contains_raw_account_session"])
+        self.assertEqual(fixture["local_ip"], "192.0.2.10")
+
+        allowed_ips = {"192.0.2.10", "198.51.100.20"}
+        for packet, decoded in zip(fixture["packets"], fixture_packets()):
+            with self.subTest(label=packet["label"]):
+                self.assertIn(packet["src_ip"], allowed_ips)
+                self.assertIn(packet["dst_ip"], allowed_ips)
+                self.assertEqual(packet["protocol"], "udp")
+                self.assertIsNone(extract_user_uid(decoded.payload))
+
+    def test_fixture_replays_permanent_and_arc_history(self):
+        fixture = load_network_fixture()
+        session = fixture_session()
+
+        permanent = annotate_groups(session.build_rows("permanent"))
+        arc = session.build_rows("arc_miracle_box")
+
+        self.assertIsNone(session.user_uid)
+        self.assertEqual(len(permanent), fixture["expected"]["permanent_records"])
+        self.assertEqual(len(arc), fixture["expected"]["arc_records"])
+        self.assertEqual(permanent[0]["uid"], fixture["expected"]["permanent_first_uid"])
+        self.assertEqual(arc[0]["uid"], fixture["expected"]["arc_first_uid"])
+        self.assertEqual(sorted({row["page"] for row in permanent}), [1, 2, 3, 4, 5])
+        self.assertEqual(sorted({row["page"] for row in arc}), [1, 2, 3, 4, 5])
+        self.assertEqual({row["timestamp_decoded"] for row in permanent}, {"2030-01-01 00:00:00"})
+        self.assertEqual({row["timestamp_decoded"] for row in arc}, {"2030-01-02 00:00:00"})
+
+        # Repeating one manufactured record prevents the replay transcript from
+        # preserving a real account's pull sequence while still exercising page
+        # ordering, timestamp ordinals, UID stability, and export behavior.
+        self.assertEqual(len({row["record_hex"] for row in permanent}), 1)
+        self.assertEqual(len({row["record_hex"] for row in arc}), 1)
+
+    def test_fixture_protocol_samples_cover_edge_cases(self):
+        points = decode_response_records(fixture_payload("limited-points-gift-1"))
+        chase = decode_response_records(fixture_payload("limited-chase-reward"))
+        batched = decode_response_records(fixture_payload("limited-batched-pages"))
+        aligned = decode_response_records(fixture_payload("limited-bitpacked-source"))
+        bitpacked = decode_response_records(fixture_payload("limited-bitpacked-response"))
+
+        self.assertEqual(len(points), 1)
+        self.assertEqual(points[0]["result_type"], "points_gift")
+        self.assertEqual(points[0]["reward_id"], "1020")
+        self.assertEqual(len(chase), 1)
+        self.assertEqual(chase[0]["result_type"], "chase_reward")
+        self.assertEqual(chase[0]["reward_id"], "Dice_ticket_01")
+        self.assertEqual(chase[0]["quantity"], 30)
+        self.assertEqual(len(batched), 10)
+        self.assertEqual(len(aligned), 10)
+        self.assertEqual(
+            [row["record_hex"] for row in bitpacked],
+            [row["record_hex"] for row in aligned],
+        )
+
+    def test_fixture_exports_sanitized_json_with_stable_uids(self):
+        session = fixture_session()
+        rows = annotate_groups(session.build_rows("permanent"))
+        export = build_export_json(rows, [])
+
+        self.assertNotIn("user_uid", export)
+        self.assertEqual(export["format_version"], 1)
+        self.assertEqual(export["records"][0]["uid"], load_network_fixture()["expected"]["permanent_first_uid"])
+        self.assertNotIn("record_hex", export["records"][0])
+        self.assertNotIn("timestamp_raw_hex", export["records"][0])

tests/test_protocol_decoding.py

@@ -0,0 +1,125 @@
+from tests.support import *  # noqa: F401,F403
+
+
+class ProtocolDecodingTests(unittest.TestCase):
+    def test_limited_selector_and_marker_decode(self):
+        request = bytearray(45)
+        request[31:35] = (4).to_bytes(4, "little")
+        request[35:39] = (4220).to_bytes(4, "little")
+        request[40:44] = (8).to_bytes(4, "little")
+        self.assertEqual(history_request_kind(bytes(request)), "limited_character")
+
+    def test_monopoly_request_allows_coalesced_trailing_payload(self):
+        request = bytearray(45)
+        request[31:35] = (25 * 4).to_bytes(4, "little")
+        request[35:39] = (4220).to_bytes(4, "little")
+        request[40:44] = (4).to_bytes(4, "little")
+        coalesced = bytes(request) + bytes.fromhex(
+            "007c669610062038461bc40100000872a34b93821a0219aa933b0a6b2ba34a6b"
+        )
+
+        self.assertEqual(history_request_kind(coalesced), "permanent")
+
+    def test_arc_request_allows_coalesced_trailing_payload(self):
+        request = bytearray(34)
+        request[24:28] = (2060).to_bytes(4, "little")
+        request[29:33] = (7 * 2).to_bytes(4, "little")
+
+        self.assertTrue(is_arc_history_request(bytes(request) + bytes(32)))
+        self.assertEqual(arc_request_page(bytes(request) + bytes(32)), 7)
+
+        response = bytearray(220)
+        response[0x50:0x54] = (4).to_bytes(4, "little")
+        response[0x54:0x58] = (20).to_bytes(4, "little")
+        response[0x58:0x5d] = bytes.fromhex("c4c0c4c000")
+        marker_offset = 0x5d
+        response[marker_offset:marker_offset + len(LIMITED_CHARACTER_MARKER)] = LIMITED_CHARACTER_MARKER
+        timestamp_raw = (2556647947780680000).to_bytes(8, "little")
+        response[marker_offset + len(LIMITED_CHARACTER_MARKER):marker_offset + len(LIMITED_CHARACTER_MARKER) + 8] = timestamp_raw
+
+        rows = decode_response_records(bytes(response))
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["reward_id"], "1010")
+        self.assertEqual(rows[0]["reward_name"], "Nanally")
+
+    def test_fixture_prefixed_points_gift_overrides_visible_dice(self):
+        decoded = decode_response_records(fixture_payload("limited-points-gift-1"))[0]
+        self.assertEqual(decoded["result_type"], "points_gift")
+        self.assertEqual(decoded["result_source_raw"], 0)
+        self.assertEqual(decoded["dice"], 0)
+        self.assertEqual(decoded["dice_raw_u32"], 0)
+        self.assertEqual(decoded["reward_id"], "1020")
+
+    def test_fixture_prefixed_chase_reward_overrides_visible_dice_and_quantity(self):
+        decoded = decode_response_records(fixture_payload("limited-chase-reward"))[0]
+
+        self.assertEqual(decoded["result_type"], "chase_reward")
+        self.assertEqual(decoded["result_source_raw"], -4)
+        self.assertEqual(decoded["dice"], -4)
+        self.assertEqual(decoded["dice_raw_u32"], -4)
+        self.assertEqual(decoded["reward_id"], "Dice_ticket_01")
+        self.assertEqual(decoded["quantity"], 30)
+
+    def test_warp_piece_chase_subrecord_without_prefix_marker_is_chase_reward(self):
+        decoded = decode_single_record(
+            "c1c4b0ccc00000000000040000003c00000010a58d957dd1a58dad95d17dc1c400"
+            "4c0000000c85c99141bdbdb17d0da185c9858dd195c90140eb2c2dd7227b23"
+        )
+
+        self.assertEqual(decoded["result_type"], "chase_reward")
+        self.assertEqual(decoded["result_source_raw"], -4)
+        self.assertEqual(decoded["dice"], -4)
+        self.assertEqual(decoded["dice_raw_u32"], -4)
+        self.assertEqual(decoded["reward_id"], "Dice_ticket_01")
+        self.assertEqual(decoded["reward_name"], "Warp Piece")
+        self.assertEqual(decoded["quantity"], 30)
+
+    def test_page_first_prefix_uses_real_dice_field(self):
+        cases = [
+            (
+                "003006000014000000040000002800000098bdc9ad7dd9a5b995010000000008000000"
+                "3c00000010a58d957dd1a58dad95d17dc1c4002800000098bdc9ad7dd9a5b995014c"
+                "0000000c85c99141bdbdb17d0da185c9858dd195c901c0dd53bd2b137b23",
+                1,
+                "fork_vine",
+            ),
+            (
+                "00c8060000140000001000000014000000c4c0d4d400000000000400000014000000"
+                "c4c0d4d400440000000c85c99141bdbdb17d3995dd49bdb1950100d929e115087b23",
+                4,
+                "1055",
+            ),
+        ]
+        for record_hex, expected_dice, reward_id in cases:
+            with self.subTest(reward_id=reward_id):
+                decoded = decode_single_record(record_hex)
+
+                self.assertEqual(decoded["dice"], expected_dice)
+                self.assertEqual(decoded["dice_raw_u32"], expected_dice * 4)
+                self.assertEqual(decoded["dice_offset_in_record"], 9)
+                self.assertEqual(decoded["result_type"], "dice")
+                self.assertEqual(decoded["reward_id"], reward_id)
+
+    def test_batched_monopoly_response_normalizes_embedded_page_header(self):
+        decoded = decode_response_records(fixture_payload("limited-batched-pages"))
+
+        self.assertEqual(len(decoded), 10)
+        self.assertEqual(decoded[5]["dice"], 4)
+        self.assertEqual(decoded[5]["result_type"], "dice")
+
+    def test_monopoly_response_parser_realigns_bit_packed_payload(self):
+        reference_rows = decode_response_records(fixture_payload("limited-bitpacked-source"))
+        decoded = decode_response_records(fixture_payload("limited-bitpacked-response"))
+
+        self.assertEqual(len(decoded), 10)
+        self.assertEqual(
+            [row["record_hex"] for row in decoded],
+            [row["record_hex"] for row in reference_rows],
+        )
+
+    def test_page_gap_warning_reports_ignored_pages(self):
+        pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (1, 2, 3, 5)]
+        run, warnings = select_continuous_run_from_page_1(pairs)
+        self.assertEqual([p[0] for p in run], [1, 2, 3])
+        self.assertEqual(warnings[0]["code"], "PAGE_GAP_DETECTED")
+        self.assertEqual(warnings[0]["ignored_pages"], [5])

tests/test_structured_protocol.py

@@ -0,0 +1,382 @@
+from __future__ import annotations
+
+import csv
+import unittest
+from dataclasses import replace
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+from nte_history_exporter.decoder.arc import (
+    build_arc_rows_from_pairs,
+    make_arc_uid,
+    parse_arc_response,
+)
+from nte_history_exporter.decoder.boundary import annotate_groups, make_uid
+from nte_history_exporter.decoder.protocol import decode_response_records
+from nte_history_exporter.decoder.structured_protocol import (
+    FORK_BLOCK_KIND,
+    FORK_MARKER,
+    MONOPOLY_BLOCK_KIND,
+    MONOPOLY_ENVELOPE_FOOTER,
+    MONOPOLY_MARKER,
+    PROTOCOL_CONSTANT,
+    ProtocolEnvelope,
+    StructuredProtocolAssembler,
+    parse_structured_blocks,
+    parse_structured_records,
+)
+from nte_history_exporter.decoder.run import build_rows_from_pairs
+from nte_history_exporter.export.csv_export import write_csv
+from nte_history_exporter.export.json_export import build_export_json
+from tests.support import fixture_payload
+
+
+STRUCTURED_TICKS = 639_131_653_353_040_000
+
+
+def fstring(value: str) -> bytes:
+    raw = value.encode("utf-8") + b"\0"
+    return len(raw).to_bytes(4, "little") + raw
+
+
+def monopoly_payload(
+    item_spec: str,
+    *,
+    ticks: int = STRUCTURED_TICKS,
+    roll_points: int = 2,
+    secondary_item_id: str = "",
+    secondary_count: int = 0,
+    pool_id: str = "CardPool_Character",
+) -> bytes:
+    row = (
+        roll_points.to_bytes(4, "little")
+        + fstring(item_spec)
+        + (0).to_bytes(4, "little")
+        + secondary_count.to_bytes(4, "little")
+        + fstring(secondary_item_id)
+        + fstring(item_spec.split(",", 1)[0])
+        + fstring(pool_id)
+        + ticks.to_bytes(8, "little")
+    )
+    return (
+        MONOPOLY_MARKER
+        + b"\0"
+        + (0).to_bytes(4, "little")
+        + len(row).to_bytes(4, "little")
+        + (1).to_bytes(4, "little")
+        + row
+    )
+
+
+def fork_payload(item_spec: str, *, ticks: int = STRUCTURED_TICKS) -> bytes:
+    row = fstring(item_spec) + fstring("ForkLottery_AnHunQu") + ticks.to_bytes(8, "little")
+    return (
+        FORK_MARKER
+        + b"\0"
+        + (0).to_bytes(4, "little")
+        + len(row).to_bytes(4, "little")
+        + (1).to_bytes(4, "little")
+        + row
+    )
+
+
+def bit_pack_after_eight_byte_header(payload: bytes, shift: int) -> bytes:
+    packed = int.from_bytes(payload, "little") << shift
+    return bytes(8) + packed.to_bytes(len(payload) + 1, "little")
+
+
+def enveloped_monopoly_payload(
+    item_spec: str,
+    *,
+    page_index: int,
+    query_high: bool,
+    pool_token: int = 256,
+    shift: int = 3,
+) -> bytes:
+    envelope = (
+        PROTOCOL_CONSTANT.to_bytes(4, "little")
+        + ((0x80000000 if query_high else 0)).to_bytes(4, "little")
+        + page_index.to_bytes(4, "little")
+        + MONOPOLY_BLOCK_KIND.to_bytes(4, "little")
+        + pool_token.to_bytes(4, "little")
+        + MONOPOLY_ENVELOPE_FOOTER.to_bytes(4, "little")
+        + b"\0\0"
+    )
+    return bit_pack_after_eight_byte_header(envelope + monopoly_payload(item_spec), shift)
+
+
+def enveloped_fork_payload(
+    item_spec: str,
+    *,
+    page_index: int,
+    query_high: bool,
+    shift: int = 3,
+) -> bytes:
+    envelope = (
+        PROTOCOL_CONSTANT.to_bytes(4, "little")
+        + ((0x80000000 if query_high else 0)).to_bytes(4, "little")
+        + page_index.to_bytes(4, "little")
+        + FORK_BLOCK_KIND.to_bytes(4, "little")
+        + b"\0"
+    )
+    return bit_pack_after_eight_byte_header(envelope + fork_payload(item_spec), shift)
+
+
+def assembled_block(item_id: str, segment_index: int, source_index: int = 0):
+    block = parse_structured_blocks(
+        monopoly_payload(item_id), "monopoly", source_index=source_index
+    )[0]
+    envelope = ProtocolEnvelope(
+        record_type="monopoly",
+        stream_key="monopoly:256",
+        page_index=segment_index // 2,
+        query_high=segment_index % 2 == 0,
+        segment_index=segment_index,
+    )
+    return replace(block, envelope=envelope)
+
+
+class StructuredProtocolTests(unittest.TestCase):
+    def test_structured_monopoly_parser_enriches_the_existing_decoder(self):
+        payload = monopoly_payload(
+            "Fashion_vehicle_1010_V008,3",
+            secondary_item_id="Dice_ticket_02",
+            secondary_count=5,
+        )
+
+        rows = decode_response_records(payload)
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["decoder_mode"], "heuristic_enriched")
+        self.assertEqual(rows[0]["reward_id"], "Fashion_vehicle_1010_V008")
+        self.assertEqual(rows[0]["reward_name"], "Tiger Incoming! - Livery")
+        self.assertEqual(rows[0]["quantity"], 3)
+        self.assertEqual(rows[0]["dice"], 2)
+        self.assertEqual(rows[0]["secondary_reward_id"], "Dice_ticket_02")
+        self.assertEqual(rows[0]["secondary_quantity"], 5)
+        self.assertEqual(rows[0]["structured_pool_id"], "CardPool_Character")
+
+    def test_structured_monopoly_parser_falls_back_when_heuristic_returns_no_rows(self):
+        payload = monopoly_payload("Dice_ticket_02,50", roll_points=0)
+
+        with patch("nte_history_exporter.decoder.protocol._decode_aligned_response_records", return_value=[]):
+            rows = decode_response_records(payload)
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["decoder_mode"], "structured_fallback")
+        self.assertEqual(rows[0]["reward_id"], "Dice_ticket_02")
+        self.assertEqual(rows[0]["quantity"], 50)
+        self.assertEqual(rows[0]["result_type"], "points_gift")
+
+    def test_structured_fork_parser_is_a_complete_fallback(self):
+        rows = parse_arc_response(fork_payload("fork_dustbin,2"))
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["decoder_mode"], "structured_fallback")
+        self.assertEqual(rows[0]["reward_id"], "fork_dustbin")
+        self.assertEqual(rows[0]["reward_name"], "Dangerous Game")
+        self.assertEqual(rows[0]["structured_pool_id"], "ForkLottery_AnHunQu")
+
+    def test_structured_parser_realigns_bit_packed_payload(self):
+        payload = bit_pack_after_eight_byte_header(monopoly_payload("1003,1"), 3)
+
+        rows = parse_structured_records(payload, "monopoly")
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0].item_id, "1003")
+        self.assertEqual(rows[0].protocol_view, "shift8:3")
+
+    def test_protocol_envelope_exposes_stream_and_segment_identity(self):
+        payload = enveloped_monopoly_payload("1003", page_index=2, query_high=False)
+
+        block = parse_structured_blocks(payload, "monopoly")[0]
+
+        self.assertIsNotNone(block.envelope)
+        self.assertEqual(block.envelope.stream_key, "monopoly:256")
+        self.assertEqual(block.envelope.page_index, 2)
+        self.assertFalse(block.envelope.query_high)
+        self.assertEqual(block.envelope.segment_index, 3)
+
+    def test_assembler_orders_segments_and_ignores_retransmissions(self):
+        assembler = StructuredProtocolAssembler()
+        segment_one = assembled_block("1010", 1)
+        assembler.add_blocks([segment_one, assembled_block("1003", 0), segment_one])
+
+        rows = assembler.rows("monopoly")
+
+        self.assertEqual([row.item_id for row in rows], ["1003", "1010"])
+        self.assertEqual([row.generation_index for row in rows], [0, 0])
+        self.assertEqual(assembler.warnings, [])
+
+    def test_assembler_never_deduplicates_blocks_without_envelopes(self):
+        assembler = StructuredProtocolAssembler()
+        block = parse_structured_blocks(monopoly_payload("1003"), "monopoly")[0]
+
+        assembler.add_blocks([block, block])
+
+        self.assertEqual([row.item_id for row in assembler.rows("monopoly")], ["1003", "1003"])
+
+    def test_new_complete_generation_replaces_old_snapshot(self):
+        assembler = StructuredProtocolAssembler()
+        assembler.add_blocks([assembled_block("1003", 0), assembled_block("1010", 1)])
+        assembler.add_blocks([assembled_block("1020", 0), assembled_block("1021", 1)])
+
+        rows = assembler.rows("monopoly")
+
+        self.assertEqual([row.item_id for row in rows], ["1020", "1021"])
+        self.assertEqual([row.generation_index for row in rows], [1, 1])
+
+    def test_partial_generation_merges_only_on_unique_overlap(self):
+        assembler = StructuredProtocolAssembler()
+        assembler.add_blocks(
+            [
+                assembled_block("1003", 0),
+                assembled_block("1010", 1),
+                assembled_block("1020", 2),
+            ]
+        )
+        assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)])
+
+        rows = assembler.rows("monopoly")
+
+        self.assertEqual([row.item_id for row in rows], ["1099", "1010", "1020"])
+        self.assertEqual(assembler.warnings, [])
+
+    def test_ambiguous_partial_generation_keeps_proven_snapshot(self):
+        assembler = StructuredProtocolAssembler()
+        assembler.add_blocks(
+            [
+                assembled_block("1003", 0),
+                assembled_block("1010", 1),
+                assembled_block("1003", 2),
+                assembled_block("1010", 3),
+            ]
+        )
+        assembler.add_blocks([assembled_block("1099", 0), assembled_block("1010", 1)])
+
+        rows = assembler.rows("monopoly")
+
+        self.assertEqual([row.item_id for row in rows], ["1003", "1010", "1003", "1010"])
+        self.assertEqual(assembler.warnings[0]["code"], "AMBIGUOUS_STRUCTURED_SNAPSHOT")
+
+    def test_pair_assembly_is_used_only_for_all_structured_fallback(self):
+        segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False)
+        segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True)
+        pairs = [
+            (2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"),
+            (1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"),
+        ]
+
+        with patch("nte_history_exporter.decoder.protocol._decode_aligned_response_records", return_value=[]):
+            rows = build_rows_from_pairs(pairs)
+        annotated = annotate_groups(rows)
+
+        self.assertEqual([row["reward_id"] for row in annotated], ["1003", "1010"])
+        self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in annotated))
+        self.assertEqual(annotated[0]["uid"], make_uid(annotated[0], 0))
+
+    def test_pair_assembly_cannot_reorder_successful_heuristic_rows(self):
+        segment_one = enveloped_monopoly_payload("1010", page_index=1, query_high=False)
+        segment_zero = enveloped_monopoly_payload("1003", page_index=0, query_high=True)
+        pairs = [
+            (2, 8, 1, 1.0, 2, 1.1, segment_one, "permanent"),
+            (1, 4, 3, 1.2, 4, 1.3, segment_zero, "permanent"),
+        ]
+
+        rows = build_rows_from_pairs(pairs)
+
+        self.assertEqual([row["reward_id"] for row in rows], ["1010", "1003"])
+        self.assertTrue(all(row["decoder_mode"] == "heuristic_enriched" for row in rows))
+        self.assertTrue(all("structured_assembly" not in row for row in rows))
+
+    def test_arc_fallback_uses_the_same_segment_assembly_and_uid_order(self):
+        segment_one = enveloped_fork_payload("fork_vine", page_index=1, query_high=False)
+        segment_zero = enveloped_fork_payload("fork_dustbin", page_index=0, query_high=True)
+        pairs = [
+            (2, 4, 1, 1.0, 2, 1.1, segment_one, "arc_miracle_box"),
+            (1, 2, 3, 1.2, 4, 1.3, segment_zero, "arc_miracle_box"),
+        ]
+
+        rows = build_arc_rows_from_pairs(pairs)
+
+        self.assertEqual([row["reward_id"] for row in rows], ["fork_dustbin", "fork_vine"])
+        self.assertTrue(all(row["structured_assembly"] == "snapshot_segments" for row in rows))
+        self.assertEqual(
+            rows[0]["uid"],
+            make_arc_uid(rows[0]["timestamp_raw_hex"], rows[0]["timestamp_group_ordinal"]),
+        )
+
+    def test_matching_structured_data_enriches_without_replacing_heuristic_identity(self):
+        heuristic_payload = fixture_payload("limited-points-gift-1")
+        original = decode_response_records(heuristic_payload)[0]
+        structured_ticks = original["timestamp_ticks"] // 4
+        combined = heuristic_payload + monopoly_payload(
+            "1020,7",
+            ticks=structured_ticks,
+            roll_points=0,
+            pool_id="CardPool_Character",
+        )
+
+        enriched = decode_response_records(combined)[0]
+
+        self.assertEqual(enriched["decoder_mode"], "heuristic_enriched")
+        self.assertEqual(enriched["quantity"], 7)
+        self.assertEqual(enriched["reward_id"], original["reward_id"])
+        self.assertEqual(enriched["timestamp_raw_hex"], original["timestamp_raw_hex"])
+        self.assertEqual(enriched["timestamp_ticks"], original["timestamp_ticks"])
+
+    def test_conflicting_structured_data_cannot_override_heuristic_record(self):
+        heuristic_payload = fixture_payload("limited-points-gift-1")
+        original = decode_response_records(heuristic_payload)[0]
+        combined = heuristic_payload + monopoly_payload(
+            "1003,99",
+            ticks=original["timestamp_ticks"] // 4,
+            roll_points=6,
+        )
+
+        decoded = decode_response_records(combined)[0]
+
+        self.assertNotIn("decoder_mode", decoded)
+        self.assertEqual(decoded["reward_id"], original["reward_id"])
+        self.assertEqual(decoded["quantity"], original["quantity"])
+        self.assertEqual(decoded["dice"], original["dice"])
+
+    def test_malformed_structured_block_fails_closed(self):
+        malformed = (
+            MONOPOLY_MARKER
+            + b"\0"
+            + (0).to_bytes(4, "little")
+            + (9999).to_bytes(4, "little")
+            + (1).to_bytes(4, "little")
+        )
+
+        self.assertEqual(decode_response_records(malformed), [])
+
+    def test_structured_diagnostics_are_debug_only(self):
+        row = decode_response_records(monopoly_payload("1003,1"))[0]
+        row.update(
+            {
+                "uid": "stable-test-uid",
+                "export_record": True,
+                "pool_group_id": "Lottery_Permanent",
+                "timestamp_group_ordinal": 0,
+            }
+        )
+
+        export_record = build_export_json([row], [])["records"][0]
+        self.assertNotIn("decoder_mode", export_record)
+        self.assertNotIn("structured_pool_id", export_record)
+        self.assertNotIn("secondary_reward_id", export_record)
+
+        with TemporaryDirectory() as tmp:
+            path = Path(tmp) / "debug.csv"
+            write_csv(path, [row])
+            with path.open(newline="", encoding="utf-8") as handle:
+                debug_record = next(csv.DictReader(handle))
+        self.assertEqual(debug_record["decoder_mode"], "heuristic_enriched")
+        self.assertEqual(debug_record["structured_pool_id"], "CardPool_Character")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_uid_compatibility.py

@@ -0,0 +1,153 @@
+from tests.support import *  # noqa: F401,F403
+
+
+class UidCompatibilityTests(unittest.TestCase):
+    def test_uid_source_matches_committed_network_fixture(self):
+        fixture = load_network_fixture()
+        rows = annotate_groups(fixture_session().build_rows("permanent"))
+        self.assertEqual(rows[0]["uid"], fixture["expected"]["permanent_first_uid"])
+
+    def test_uid_uses_pool_timestamp_and_ordinal_only(self):
+        row = {
+            "pool_group_id": "Lottery_LimitedCharacter",
+            "timestamp_raw_hex": "40e93247c3097b23",
+            "dice": 5,
+            "reward_key_hex": "10a58d957dd1a58dad95d17dc1c800",
+            "quantity": 50,
+        }
+        changed_content = {
+            **row,
+            "dice": 1,
+            "reward_key_hex": "98bdc9ad7dd9a5b99501",
+            "quantity": 1,
+        }
+        changed_pool = {**row, "pool_group_id": "Lottery_Permanent"}
+
+        self.assertEqual(make_uid(row, 0), "74a9ef4aacde549dfe8e8e7cc6ddd65b")
+        self.assertEqual(make_uid(changed_content, 0), make_uid(row, 0))
+        self.assertNotEqual(make_uid(changed_pool, 0), make_uid(row, 0))
+        self.assertNotEqual(make_uid(row, 1), make_uid(row, 0))
+
+    def test_pages_1_to_5_exports_every_row(self):
+        annotated = annotate_groups(fixture_session().build_rows("permanent"))
+
+        exported = [row for row in annotated if row["export_record"] is True]
+
+        # Every decoded row is exported; boundary groups are never dropped.
+        self.assertEqual(len(annotated), 25)
+        self.assertEqual(len(exported), 25)
+
+    @staticmethod
+    def _synthetic_row(page, timestamp_hex, result_type):
+        return {
+            "page": page,
+            "timestamp_raw_hex": timestamp_hex,
+            "timestamp_decoded": f"ts-{timestamp_hex}",
+            "result_type": result_type,
+            "dice": 4 if result_type == "dice" else 0,
+            "reward_key_hex": "10a58d9539bdc9b585b101",
+            "quantity": 1,
+        }
+
+    def test_oldest_group_with_partial_dice_count_exports_without_warning(self):
+        rows = [self._synthetic_row(1, "aa", "dice") for _ in range(5)]
+        rows += [self._synthetic_row(2, "bb", "dice") for _ in range(5)]
+        rows += [self._synthetic_row(3, "bb", "dice") for _ in range(4)]
+        rows += [self._synthetic_row(3, "bb", "points_gift")]
+
+        annotated = annotate_groups(rows)
+        exported = [row for row in annotated if row["export_record"] is True]
+
+        # Oldest group is a partially captured 10-pull on a full final page. Its
+        # captured prefix is ordinal-stable, so it is exported with stable UIDs.
+        self.assertEqual(len(exported), 15)
+        oldest = [row for row in annotated if row["timestamp_raw_hex"] == "bb"]
+        self.assertTrue(all(row["uid"] for row in oldest))
+        self.assertTrue(all(row["uid_status"] == "stable" for row in oldest))
+        self.assertEqual([row["timestamp_group_ordinal"] for row in oldest], list(range(10)))
+
+    def test_incomplete_oldest_prefix_keeps_stable_uids(self):
+        full = [self._synthetic_row(1, "aa", "dice") for _ in range(5)]
+        full += [self._synthetic_row(2, "bb", "dice") for _ in range(3)]
+        full += [self._synthetic_row(3, "bb", "dice") for _ in range(2)]
+        truncated = [r for r in full if r["page"] in (1, 2)]
+
+        full_rows = annotate_groups([dict(r) for r in full])
+        trunc_rows = annotate_groups([dict(r) for r in truncated])
+
+        full_uids = [r["uid"] for r in full_rows if r["timestamp_raw_hex"] == "bb"][:3]
+        trunc_uids = [r["uid"] for r in trunc_rows if r["timestamp_raw_hex"] == "bb"]
+        # Capturing only the first 3 of a 5-record oldest group yields the same
+        # UIDs those rows have in the full capture.
+        self.assertEqual(len(trunc_uids), 3)
+        self.assertEqual(trunc_uids, full_uids)
+
+    def test_oldest_group_with_ten_dice_exports_on_full_final_page(self):
+        rows = [self._synthetic_row(1, "aa", "dice") for _ in range(5)]
+        rows += [self._synthetic_row(2, "bb", "dice") for _ in range(5)]
+        rows += [self._synthetic_row(3, "bb", "dice") for _ in range(5)]
+
+        annotated = annotate_groups(rows)
+        exported = [row for row in annotated if row["export_record"] is True]
+
+        self.assertEqual(len(exported), 15)
+
+    def test_run_selection_anchors_to_page_1_and_keeps_newest(self):
+        # Page 2's response was lost: captured pages 1, 3, 4, 5.
+        pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (1, 3, 4, 5)]
+        run, warnings = select_continuous_run_from_page_1(pairs)
+
+        # The page-1 run (just page 1, the newest history) is kept; later pages are
+        # ignored with a gap warning, never silently discarding page 1.
+        self.assertEqual([p[0] for p in run], [1])
+        self.assertEqual(len(warnings), 1)
+        self.assertEqual(warnings[0]["code"], "PAGE_GAP_DETECTED")
+        self.assertEqual(warnings[0]["ignored_pages"], [3, 4, 5])
+
+    def test_run_selection_warns_when_page_1_missing(self):
+        pairs = [(p, p * 2, 0, 0, 0, 0, b"", "permanent") for p in (3, 4, 5)]
+        run, warnings = select_continuous_run_from_page_1(pairs)
+
+        self.assertEqual([p[0] for p in run], [3, 4, 5])
+        self.assertEqual(warnings[0]["code"], "DID_NOT_START_AT_PAGE_1")
+
+    def test_committed_network_scan_exports_all_rows(self):
+        fixture = load_network_fixture()
+        annotated = annotate_groups(fixture_session().build_rows("permanent"))
+        exported = [row for row in annotated if row["export_record"] is True]
+
+        self.assertEqual(len(annotated), fixture["expected"]["permanent_records"])
+        self.assertEqual(len(exported), fixture["expected"]["permanent_records"])
+
+    def test_monopoly_uid_compatibility_vectors_are_frozen(self):
+        vectors = [
+            ("Lottery_Permanent", "0000000000000000", 0, "5da57468adb23fcfde530fd849a25767"),
+            ("Lottery_Permanent", "00a243eb689b7a23", 0, "8f9fb9bc92a867f6f41e5839e94c7991"),
+            ("Lottery_Permanent", "00a243eb689b7a23", 1, "6a0bea71d091b9b7063780fcc86238d5"),
+            ("Lottery_LimitedCharacter", "00a243eb689b7a23", 0, "b8f5d3c27315c70ceafbe8eb2bab0fc2"),
+            ("Lottery_LimitedCharacter", "ffffffffffffffff", 9, "66916e1f077fdf7664f0f0edc4c64eb8"),
+        ]
+
+        for pool_group_id, timestamp_raw_hex, ordinal, expected_uid in vectors:
+            with self.subTest(
+                pool_group_id=pool_group_id,
+                timestamp=timestamp_raw_hex,
+                ordinal=ordinal,
+            ):
+                row = {
+                    "pool_group_id": pool_group_id,
+                    "timestamp_raw_hex": timestamp_raw_hex,
+                }
+                self.assertEqual(make_uid(row, ordinal), expected_uid)
+
+    def test_arc_uid_compatibility_vectors_are_frozen(self):
+        vectors = [
+            ("0000000000000000", 0, "faa0b07cefd71758d4a8d64bc2a2f78e"),
+            ("00a243eb689b7a23", 0, "7a7ef75281fcced043f691d269477cfa"),
+            ("00a243eb689b7a23", 1, "f798c076ea1d137a08fe62df859e8a7f"),
+            ("ffffffffffffffff", 9, "d114cc6c313a15f55ca12910801d78ea"),
+        ]
+
+        for timestamp_raw_hex, ordinal, expected_uid in vectors:
+            with self.subTest(timestamp=timestamp_raw_hex, ordinal=ordinal):
+                self.assertEqual(make_arc_uid(timestamp_raw_hex, ordinal), expected_uid)

tools/update_mappings.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+    sys.path.insert(0, str(SRC))
+
+from nte_history_exporter.mapping_update import (  # noqa: E402
+    DEFAULT_SOURCE_REF,
+    MappingUpdateError,
+    apply_update,
+    build_mapping_update,
+    load_assets,
+    load_current_mappings,
+    write_update,
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Synchronise reward mappings from the authoritative NTE_Assets tables."
+    )
+    parser.add_argument(
+        "--assets-root",
+        type=Path,
+        help="local NTE_Assets checkout (downloads the required tables when omitted)",
+    )
+    parser.add_argument(
+        "--source-ref",
+        default=DEFAULT_SOURCE_REF,
+        help="NTE_Assets branch, tag, or commit to download (default: main)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=ROOT / "build" / "mapping-update",
+        help="directory for staged mappings and the review report",
+    )
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="replace reward mapping files after staging and validation",
+    )
+    parser.add_argument(
+        "--check",
+        action="store_true",
+        help="exit 1 when the generated snapshot differs from local mappings",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        assets = load_assets(assets_root=args.assets_root, source_ref=args.source_ref)
+        current = load_current_mappings(ROOT / "mappings")
+        result = build_mapping_update(current, assets)
+        write_update(result, args.output_dir)
+        if args.apply:
+            apply_update(result, ROOT / "mappings")
+    except (MappingUpdateError, OSError) as exc:
+        print(f"mapping update failed: {exc}", file=sys.stderr)
+        return 2
+
+    changes = result.report["changes"]
+    print(f"source: {assets.source}")
+    if assets.source_ref:
+        print(f"source ref: {assets.source_ref}")
+    print(f"staged: {args.output_dir}")
+    print(
+        "changes: "
+        f"{changes['additions']} additions, "
+        f"{changes['updates']} updates, "
+        f"{changes['deletions']} deletions"
+    )
+    if args.apply:
+        print("applied: mappings/arcs.json, mappings/characters.json, mappings/items.json")
+    return 1 if args.check and result.change_count else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())