Added mappings updater

Implemented a mappings updater script that uses the Waifus-Grace/NTE_Assets repo to export all the required mappings.
This commit is contained in:
Golumpa 2026-07-15 19:06:47 +01:00
parent dd07335d39
commit 042d47e3d4
17 changed files with 980 additions and 39 deletions

136
.github/workflows/update-mappings.yml vendored Normal file
View file

@ -0,0 +1,136 @@
name: Update reward mappings
on:
workflow_dispatch:
inputs:
source_ref:
description: NTE_Assets branch, tag, or commit
required: true
default: main
type: string
permissions:
contents: write
pull-requests: write
concurrency:
group: update-reward-mappings
cancel-in-progress: false
jobs:
update:
name: Rebuild mappings from NTE_Assets
runs-on: ubuntu-latest
env:
AUTOMATION_BRANCH: dev/automated-mapping-update
NTE_ASSETS_REF: ${{ inputs.source_ref }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- name: Checkout source code
uses: actions/checkout@v5
with:
fetch-depth: 0
ref: ${{ github.event.repository.default_branch }}
- name: Get Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install project
run: python -m pip install -e .
- name: Rebuild reward mappings
run: python tools/update_mappings.py --source-ref "$NTE_ASSETS_REF" --apply
- name: Run tests
run: python -m unittest discover -s tests
- name: Summarize mapping changes
id: changes
shell: bash
run: |
python - <<'PY'
import json
import os
from pathlib import Path
report = json.loads(Path("build/mapping-update/mapping-update-report.json").read_text())
changes = report["changes"]
changed = sum(changes.values()) > 0
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
output.write(f"changed={'true' if changed else 'false'}\n")
with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as summary:
summary.write("## NTE_Assets mapping update\n\n")
summary.write(f"Source ref: `{report['source_ref']}`\n\n")
summary.write(f"- Additions: {changes['additions']}\n")
summary.write(f"- Updates: {changes['updates']}\n")
summary.write(f"- Deletions: {changes['deletions']}\n")
PY
- name: Upload review report
uses: actions/upload-artifact@v6
with:
name: mapping-update-report
path: build/mapping-update/mapping-update-report.json
- name: Commit and push mapping changes
if: steps.changes.outputs.changed == 'true'
shell: bash
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -B "$AUTOMATION_BRANCH"
git add mappings/arcs.json mappings/characters.json mappings/items.json
git commit -m "chore: update reward mappings from NTE_Assets"
git push --force origin "HEAD:refs/heads/$AUTOMATION_BRANCH"
- name: Open or update pull request
if: steps.changes.outputs.changed == 'true'
shell: bash
env:
BASE_BRANCH: ${{ github.event.repository.default_branch }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
python - <<'PY'
import json
import os
from pathlib import Path
report = json.loads(Path("build/mapping-update/mapping-update-report.json").read_text())
changes = report["changes"]
body = (
"Automated authoritative reward-mapping snapshot from "
"[Waifus-Grace/NTE_Assets](https://github.com/Waifus-Grace/NTE_Assets).\n\n"
f"Source ref: `{report['source_ref']}`\n\n"
f"- {changes['additions']} additions\n"
f"- {changes['updates']} updates\n"
f"- {changes['deletions']} deletions\n\n"
"The workflow rebuilt all reward mappings, resolved English names through "
"`Localization/en/game.json`, and passed the complete test suite. Pool mapping "
"files and UID inputs were not changed.\n\n"
f"[Workflow run]({os.environ['RUN_URL']})\n"
)
Path("build/mapping-update/pull-request-body.md").write_text(body, encoding="utf-8")
PY
pr_number="$(gh pr list \
--head "$AUTOMATION_BRANCH" \
--base "$BASE_BRANCH" \
--state open \
--json number \
--jq '.[0].number // empty')"
if [ -n "$pr_number" ]; then
gh pr edit "$pr_number" \
--title "chore: update reward mappings from NTE_Assets" \
--body-file build/mapping-update/pull-request-body.md
else
gh pr create \
--head "$AUTOMATION_BRANCH" \
--base "$BASE_BRANCH" \
--title "chore: update reward mappings from NTE_Assets" \
--body-file build/mapping-update/pull-request-body.md
fi

View file

@ -163,6 +163,26 @@ The exporter automatically includes the shareable NTE user UID when it appears i
> [!TIP]
> For reliable deduplication, start from page 1 and scroll through the pages. If you only want pages 1–5, scroll through to page 6 as well just to be on the safe side.
## Mapping maintenance
Reward metadata can be rebuilt directly from the latest NTE_Assets tables and
English translation files. The reward snapshot may change IDs while UID inputs
remain untouched:
Run **Update reward mappings** from the GitHub Actions tab to generate and test
the snapshot in a reviewable pull request, or run it locally:
```powershell
python tools/update_mappings.py
```
This rebuilds reviewable reward-map candidates directly from NTE_Assets under
`build/mapping-update/`; committed mappings are untouched unless `--apply` is
explicitly supplied. The snapshot may include additions, updates, and removals,
while pool mappings and UID inputs remain untouched. See
[Reward mapping updates](docs/mapping-updates.md) for the source rules and
review workflow.
## Privacy
> [!CAUTION]

95
docs/mapping-updates.md Normal file
View file

@ -0,0 +1,95 @@
# Reward mapping updates
`tools/update_mappings.py` rebuilds `arcs.json`, `characters.json`, and
`items.json` directly from the current
[`Waifus-Grace/NTE_Assets`](https://github.com/Waifus-Grace/NTE_Assets)
tables. No other exporter or intermediate mapping format is used.
English display names come from `Localization/en/game.json`. The data tables
provide localization namespace/key references and quality metadata; their
embedded `LocalizedString` values are deliberately ignored.
The generated files are an authoritative snapshot, not an additive merge.
When NTE_Assets removes an ID, changes its casing, or updates its metadata, the
staged mapping reflects that change. This is intentional for a live-service
game whose asset tables evolve over time.
## Review workflow
### GitHub Actions
Run **Update reward mappings** from the repository's Actions tab. The optional
`source_ref` input accepts an NTE_Assets branch, tag, or commit and defaults to
`main`.
The workflow rebuilds the mappings, runs the complete test suite, uploads the
JSON review report, and opens a pull request from
`dev/automated-mapping-update`. Running it again updates the same open pull
request. If the authoritative snapshot has not changed, it reports zero changes
and does not create a commit or pull request.
The repository setting **Allow GitHub Actions to create and approve pull
requests** must be enabled for automatic pull-request creation. The workflow
uses only the repository-scoped `GITHUB_TOKEN` and grants it `contents: write`
and `pull-requests: write` permissions.
### Local
Stage the latest candidate files without touching the committed mappings:
```powershell
python tools/update_mappings.py
```
The required JSON tables are downloaded from the `main` branch by default.
Pin a tag, branch, or commit for reproducible review:
```powershell
python tools/update_mappings.py --source-ref e9752e5963103529f0d683fd0aed4752b7dfad78
```
To use an existing local checkout without network access:
```powershell
python tools/update_mappings.py --assets-root path\to\NTE_Assets
```
The default output is `build/mapping-update/` and contains the three candidate
mapping files plus `mapping-update-report.json`. The report lists additions,
updates, and deletions by file and records SHA-256 hashes for every source
table.
After reviewing the report and diff, apply the validated snapshot explicitly:
```powershell
python tools/update_mappings.py --apply
python -m pytest -q
```
For CI, `--check` exits with status 1 whenever the generated snapshot differs
from the committed reward mappings. It still writes the staged artifacts.
## Source rules
- Every Arc in `DT_ForkItemData` is emitted to `arcs.json`.
- Every character in `DT_Character` is emitted to `characters.json`.
- Other pull rewards are selected by `GachaIllustrate` and resolved to the
inventory tables for quality and localization keys. Appearance-table IDs
provide canonical casing for glider rewards.
- Names are resolved strictly through `Localization/en/game.json`. A missing or
ambiguous key fails the update instead of falling back to a DT value.
- Character-awakening illustration entries are not independent pull rewards
and are excluded.
- Orange, purple, and blue item qualities map to `S`, `A`, and `B`.
## UID compatibility boundary
This updater reads and writes only the three reward mapping files. It never
reads or writes the permanent, limited, beginner, or Arc pool mapping files.
Banner IDs, timestamps, record ordering, format version, and all UID inputs are
therefore unchanged by mapping synchronization.
Reward IDs and display metadata may change or disappear when the authoritative
NTE_Assets snapshot changes. Existing exported history files remain unchanged;
new exports describe rewards using the current asset snapshot. UID stability is
independent of reward display mappings.

View file

@ -1,38 +1,36 @@
{
"fork_dustbin": {"name": "Dangerous Game", "rank": "B"},
"fork_vine": {"name": "Be Happy", "rank": "B"},
"fork_nonos": {"name": "First Step to Success", "rank": "B"},
"fork_appliance": {"name": "\"Real Music\"", "rank": "B"},
"fork_Prokaryon": {"name": "Us.", "rank": "B"},
"fork_PaperPlane": {"name": "Clear Skies", "rank": "A"},
"fork_mofeikesi": {"name": "Good Boy's Grand Adventure", "rank": "S"},
"fork_wuhuakuang": {"name": "The Forgotten", "rank": "A"},
"fork_whuakuang": {"name": "The Forgotten", "rank": "A"},
"fork_jingmotingyuan": {"name": "Camellia Society", "rank": "S"},
"fork_Kite": {"name": "Watch Your Heads!", "rank": "A"},
"fork_yuren": {"name": "Umbrella", "rank": "A"},
"fork_yaodao": {"name": "Drawn Blade", "rank": "A"},
"fork_Arachne": {"name": "Eternal Waltz", "rank": "S"},
"fork_BitGame": {"name": "Blow up the Crowd", "rank": "S"},
"fork_BitterCake": {"name": "The Good, The Bad, The Bitter", "rank": "A"},
"fork_BlackBook": {"name": "Youthful Fantasy", "rank": "S"},
"fork_Baozhatang": {"name": "Fluff of Fearlessness", "rank": "S"},
"fork_BlastCandy": {"name": "Fluff of Fearlessness", "rank": "S"},
"fork_bopu": {"name": "Cosmos Daze, Wild Reverie", "rank": "A"},
"fork_Quanjitang": {"name": "Fluff of Fortitude", "rank": "S"},
"fork_BoxingCandy": {"name": "Fluff of Fortitude", "rank": "S"},
"fork_Butterfly": {"name": "Reality Refuge", "rank": "S"},
"fork_Castle": {"name": "Call of the Twisted City", "rank": "A"},
"fork_Qiaoqiao": {"name": "Time Bandit", "rank": "A"},
"fork_Crowbar": {"name": "Time Bandit", "rank": "A"},
"fork_Door": {"name": "The Wrong Gate", "rank": "S"},
"fork_dustbin": {"name": "Dangerous Game", "rank": "B"},
"fork_GoldWool": {"name": "What's Desired", "rank": "S"},
"fork_jiaojuan": {"name": "Shiny Days", "rank": "A"},
"fork_jingmotingyuan": {"name": "Camellia Society", "rank": "S"},
"fork_Kite": {"name": "Watch Your Heads!", "rank": "A"},
"fork_KnightCandy": {"name": "Fluff of Ferocity", "rank": "S"},
"fork_koinobori": {"name": "A Time Will Come", "rank": "A"},
"fork_lingganzhongjiezhe": {"name": "Mind Royale", "rank": "A"},
"fork_LunarPhase": {"name": "Blushing Mirage", "rank": "S"},
"fork_mamen": {"name": "Contemplative Cat", "rank": "S"},
"fork_mofeikesi": {"name": "Good Boy's Grand Adventure", "rank": "S"},
"fork_moon": {"name": "Stellar Veil", "rank": "S"},
"fork_MotorCandy": {"name": "Fluff of Fleetness", "rank": "S"},
"fork_Nakupeda": {"name": "Your Happiness is Priceless", "rank": "S"},
"fork_NestBird": {"name": "Tears Beneath the Mask", "rank": "S"},
"fork_nonos": {"name": "First Step to Success", "rank": "B"},
"fork_oulaquantao": {"name": "Oraora!", "rank": "A"},
"fork_PaperPlane": {"name": "Clear Skies", "rank": "A"},
"fork_PoliceRat": {"name": "Hethereau's Keeper", "rank": "S"},
"fork_Prokaryon": {"name": "Us.", "rank": "B"},
"fork_rishi": {"name": "Day Off", "rank": "S"},
"fork_Rose": {"name": "The Last Rose", "rank": "S"},
"fork_snowman": {"name": "The Fools' Spring", "rank": "A"},
@ -41,9 +39,11 @@
"fork_TigerTally": {"name": "Ready-Ready", "rank": "S"},
"fork_Time": {"name": "Marching Beyond Time", "rank": "S"},
"fork_tuansanlang": {"name": "The Great Thief", "rank": "A"},
"fork_vine": {"name": "Be Happy", "rank": "B"},
"fork_Whale": {"name": "Song of the Whale", "rank": "S"},
"fork_worldrain": {"name": "The Rain That Shook the World", "rank": "S"},
"fork_Wushoutieyu": {"name": "Raging Flames", "rank": "S"},
"fork_GoldWool": {"name": "What's Desired", "rank": "S"},
"fork_LunarPhase": {"name": "Blushing Mirage", "rank": "S"}
"fork_wuhuakuang": {"name": "The Forgotten", "rank": "A"},
"fork_wushoutieyu": {"name": "Raging Flames", "rank": "S"},
"fork_yaodao": {"name": "Drawn Blade", "rank": "A"},
"fork_yuren": {"name": "Umbrella", "rank": "A"}
}

View file

@ -19,5 +19,6 @@
"1070": {"name": "Aurelia", "rank": "A"},
"1071": {"name": "Chaos", "rank": "S"},
"1073": {"name": "Chiz", "rank": "S"},
"1075": {"name": "Iroi", "rank": "S"},
"1076": {"name": "Shinku", "rank": "S"}
}

View file

@ -1,26 +1,25 @@
{
"DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"},
"Dice_ticket_01": {"type": "item", "name": "Warp Piece", "rank": "S"},
"Dice_ticket_02": {"type": "item", "name": "Lost Piece", "rank": "A"},
"Dicelimite": {"type": "item", "name": "Solid Dice", "rank": "S"},
"Fashion_Glide_1010": {"type": "cosmetic", "name": "Glider - Underboss-of-the-Underboss", "rank": "A"},
"Fashion_vehicle_1010_V008": {"type": "cosmetic", "name": "Mod Parts - Tiger Incoming! - Livery", "rank": "S"},
"Fashion_character_1010": {"type": "cosmetic", "name": "Character Skin - Phoenix Kick", "rank": "S"},
"Fashion_Glide_1052": {"type": "cosmetic", "name": "Glider - Orchid Breeze", "rank": "A"},
"Fashion_vehicle_1052_V024": {"type": "cosmetic", "name": "Mod Parts - Autumn Haze - Livery", "rank": "S"},
"Fashion_character_1052_01": {"type": "cosmetic", "name": "Character Skin - Priceless Orchid", "rank": "S"},
"Fashion_Glide_1004": {"type": "cosmetic", "name": "Glider - Tomato Duo", "rank": "A"},
"Fashion_vehicle_1004_V021": {"type": "cosmetic", "name": "Mod Parts - Tomato Cruise - Livery", "rank": "S"},
"Fashion_character_1004_01": {"type": "cosmetic", "name": "Character Skin - Gilded Rhapsody", "rank": "S"},
"Fashion_Glide_1071": {"type": "cosmetic", "name": "Glider - Skyrider", "rank": "A"},
"Fashion_vehicle_1071_V010": {"type": "cosmetic", "name": "Mod Parts - Novis ST-X 950 - Livery", "rank": "S"},
"Fashion_character_1071_01": {"type": "cosmetic", "name": "Character Skin - Clear Skies", "rank": "S"},
"Fashion_Glide_1076": {"type": "cosmetic", "name": "Glider - Overcast Canopy", "rank": "A"},
"Fashion_vehicle_1076_V024": {"type": "cosmetic", "name": "Mod Parts - Hidden Dragon - Livery", "rank": "S"},
"Fashion_character_1076_01": {"type": "cosmetic", "name": "Character Skin - Student of Terrasea", "rank": "S"}
"DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"},
"Fashion_character_1004_01": {"type": "cosmetic", "name": "Gilded Rhapsody", "rank": "S"},
"Fashion_character_1010": {"type": "cosmetic", "name": "Phoenix Kick", "rank": "S"},
"Fashion_character_1052_01": {"type": "cosmetic", "name": "Priceless Orchid", "rank": "S"},
"Fashion_character_1071_01": {"type": "cosmetic", "name": "Clear Skies", "rank": "S"},
"Fashion_character_1075_01": {"type": "cosmetic", "name": "Summer Dream", "rank": "S"},
"Fashion_character_1076_01": {"type": "cosmetic", "name": "Student of Terrasea", "rank": "S"},
"Fashion_Glide_1004": {"type": "cosmetic", "name": "Tomato Duo", "rank": "A"},
"Fashion_Glide_1010": {"type": "cosmetic", "name": "Underboss-of-the-Underboss", "rank": "A"},
"Fashion_Glide_1052": {"type": "cosmetic", "name": "Orchid Breeze", "rank": "A"},
"Fashion_Glide_1071": {"type": "cosmetic", "name": "Skyrider", "rank": "A"},
"Fashion_glide_1075": {"type": "cosmetic", "name": "Skyrider", "rank": "A"},
"Fashion_Glide_1076": {"type": "cosmetic", "name": "Overcast Canopy", "rank": "A"},
"Fashion_vehicle_1004_V021": {"type": "cosmetic", "name": "Tomato Cruise - Livery", "rank": "S"},
"Fashion_vehicle_1010_V008": {"type": "cosmetic", "name": "Tiger Incoming! - Livery", "rank": "S"},
"Fashion_vehicle_1052_V024": {"type": "cosmetic", "name": "Autumn Haze - Livery", "rank": "S"},
"Fashion_vehicle_1071_V010": {"type": "cosmetic", "name": "Hound Blitz - Livery", "rank": "S"},
"Fashion_vehicle_1075_V006": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"},
"Fashion_vehicle_1076_V024": {"type": "cosmetic", "name": "Hidden Dragon - Livery", "rank": "S"},
"Fashion_vehicleSkin_1052_V016": {"type": "cosmetic", "name": "Golden Breeze - Livery", "rank": "S"}
}

View file

@ -0,0 +1,387 @@
from __future__ import annotations
import hashlib
import json
import os
import tempfile
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from typing import Any
NTE_ASSETS_REPOSITORY = "https://github.com/Waifus-Grace/NTE_Assets"
DEFAULT_SOURCE_REF = "main"
ASSET_TABLES = {
"characters": "DataTable/Character/DT_Character.json",
"arcs": "DataTable/Fork/DT_ForkItemData.json",
"inventory": "DataTable/Inventory/DT_ItemConfig.json",
"capital_inventory": "DataTable/Inventory/DT_CapitalItemConfig.json",
"appearances": "DataTable/Character/Appearance/DT_AppearanceData.json",
"illustrations": "DataTable/Gacha/GachaIllustrate.json",
"localization": "Localization/en/game.json",
}
REWARD_MAPPING_FILES = ("arcs.json", "characters.json", "items.json")
RANK_BY_QUALITY = {
"EItemQuality::ITEM_QUALITY_ORANGE": "S",
"EItemQuality::ITEM_QUALITY_PURPLE": "A",
"EItemQuality::ITEM_QUALITY_BLUE": "B",
}
class MappingUpdateError(ValueError):
pass
@dataclass(frozen=True)
class AssetSource:
tables: dict[str, dict[str, Any]]
source: str
source_ref: str | None
sha256: str
file_sha256: dict[str, str]
@dataclass(frozen=True)
class MappingUpdateResult:
mappings: dict[str, dict[str, dict[str, Any]]]
report: dict[str, Any]
@property
def change_count(self) -> int:
changes = self.report["changes"]
return changes["additions"] + changes["updates"] + changes["deletions"]
def load_assets(*, assets_root: Path | None = None, source_ref: str = DEFAULT_SOURCE_REF) -> AssetSource:
tables: dict[str, dict[str, Any]] = {}
file_hashes: dict[str, str] = {}
combined = hashlib.sha256()
for label, relative_path in ASSET_TABLES.items():
if assets_root is not None:
path = assets_root / Path(relative_path)
try:
raw = path.read_bytes()
except OSError as exc:
raise MappingUpdateError(f"cannot read NTE_Assets table {path}: {exc}") from exc
else:
url = f"https://raw.githubusercontent.com/Waifus-Grace/NTE_Assets/{source_ref}/{relative_path}"
request = urllib.request.Request(url, headers={"User-Agent": "nte-history-exporter-mapping-update"})
try:
with urllib.request.urlopen(request, timeout=60) as response:
raw = response.read()
except OSError as exc:
raise MappingUpdateError(f"cannot download NTE_Assets table {relative_path}: {exc}") from exc
try:
document = json.loads(raw.decode("utf-8-sig"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise MappingUpdateError(f"invalid NTE_Assets JSON in {relative_path}: {exc}") from exc
tables[label] = (
_extract_localization(document, relative_path)
if label == "localization"
else _extract_rows(document, relative_path)
)
digest = hashlib.sha256(raw).hexdigest()
file_hashes[relative_path] = digest
combined.update(relative_path.encode("utf-8"))
combined.update(b"\0")
combined.update(raw)
source = str(assets_root.resolve()) if assets_root is not None else NTE_ASSETS_REPOSITORY
return AssetSource(
tables=tables,
source=source,
source_ref=None if assets_root is not None else source_ref,
sha256=combined.hexdigest(),
file_sha256=file_hashes,
)
def load_current_mappings(directory: Path) -> dict[str, dict[str, dict[str, Any]]]:
mappings = {}
for filename in REWARD_MAPPING_FILES:
path = directory / filename
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise MappingUpdateError(f"cannot read {path}: {exc}") from exc
if not isinstance(value, dict):
raise MappingUpdateError(f"{path} must contain an object")
mappings[filename] = value
validate_mappings(mappings)
return mappings
def build_mapping_update(
current: dict[str, dict[str, dict[str, Any]]],
assets: AssetSource,
) -> MappingUpdateResult:
validate_mappings(current)
translations = _build_translation_index(assets.tables["localization"])
output = {
"arcs.json": _build_primary_mapping(assets.tables["arcs"], "arc", translations),
"characters.json": _build_primary_mapping(assets.tables["characters"], "character", translations),
"items.json": _build_item_mapping(assets.tables, translations),
}
validate_mappings(output)
additions: dict[str, list[str]] = {}
updates: dict[str, list[str]] = {}
deletions: dict[str, list[str]] = {}
for filename in REWARD_MAPPING_FILES:
old = current[filename]
new = output[filename]
additions[filename] = sorted(new.keys() - old.keys(), key=str.casefold)
deletions[filename] = sorted(old.keys() - new.keys(), key=str.casefold)
updates[filename] = sorted(
(item_id for item_id in old.keys() & new.keys() if old[item_id] != new[item_id]),
key=str.casefold,
)
report = {
"schema_version": 2,
"source": assets.source,
"source_ref": assets.source_ref,
"source_sha256": assets.sha256,
"source_file_sha256": assets.file_sha256,
"safety": {
"authoritative_snapshot": True,
"deletions_allowed": True,
"pool_mappings_touched": False,
"uid_inputs_touched": False,
},
"changes": {
"additions": sum(map(len, additions.values())),
"updates": sum(map(len, updates.values())),
"deletions": sum(map(len, deletions.values())),
},
"additions_by_file": additions,
"updates_by_file": updates,
"deletions_by_file": deletions,
"output_counts": {filename: len(entries) for filename, entries in output.items()},
}
return MappingUpdateResult(output, report)
def write_update(result: MappingUpdateResult, directory: Path) -> None:
directory.mkdir(parents=True, exist_ok=True)
for filename in REWARD_MAPPING_FILES:
_atomic_write(directory / filename, _dump_mapping(result.mappings[filename]))
report_text = json.dumps(result.report, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
_atomic_write(directory / "mapping-update-report.json", report_text)
def apply_update(result: MappingUpdateResult, directory: Path) -> None:
"""Replace all reward maps as one transaction, rolling back on failure."""
directory.mkdir(parents=True, exist_ok=True)
originals: dict[Path, bytes | None] = {}
written: list[Path] = []
for filename in REWARD_MAPPING_FILES:
path = directory / filename
originals[path] = path.read_bytes() if path.exists() else None
try:
for filename in REWARD_MAPPING_FILES:
path = directory / filename
_atomic_write(path, _dump_mapping(result.mappings[filename]))
written.append(path)
except OSError:
for path in reversed(written):
original = originals[path]
if original is None:
path.unlink(missing_ok=True)
else:
_atomic_write_bytes(path, original)
raise
def validate_mappings(mappings: dict[str, dict[str, dict[str, Any]]]) -> None:
if set(mappings) != set(REWARD_MAPPING_FILES):
raise MappingUpdateError(f"mapping set must be exactly {REWARD_MAPPING_FILES}")
locations: dict[str, str] = {}
folded: dict[str, str] = {}
for filename in REWARD_MAPPING_FILES:
entries = mappings[filename]
if not isinstance(entries, dict):
raise MappingUpdateError(f"{filename} must contain an object")
for item_id, meta in entries.items():
if not isinstance(item_id, str) or not item_id:
raise MappingUpdateError(f"{filename} contains an invalid item ID")
if item_id in locations:
raise MappingUpdateError(f"item ID {item_id} appears in {locations[item_id]} and {filename}")
locations[item_id] = filename
case_key = item_id.casefold()
if case_key in folded and folded[case_key] != item_id:
raise MappingUpdateError(f"case-insensitive duplicate IDs: {folded[case_key]} and {item_id}")
folded[case_key] = item_id
if not isinstance(meta, dict):
raise MappingUpdateError(f"{filename}:{item_id} must be an object")
name = meta.get("name")
if not isinstance(name, str) or not name.strip():
raise MappingUpdateError(f"{filename}:{item_id}.name must be a non-empty string")
rank = meta.get("rank")
if filename == "arcs.json" and rank not in {"S", "A", "B"}:
raise MappingUpdateError(f"{filename}:{item_id}.rank must be S, A, or B")
if filename == "characters.json" and rank not in {"S", "A"}:
raise MappingUpdateError(f"{filename}:{item_id}.rank must be S or A")
if filename == "items.json":
if meta.get("type") not in {"item", "cosmetic"}:
raise MappingUpdateError(f"{filename}:{item_id}.type must be item or cosmetic")
if rank not in {"S", "A", "B"}:
raise MappingUpdateError(f"{filename}:{item_id}.rank must be S, A, or B")
def _extract_rows(document: Any, relative_path: str) -> dict[str, Any]:
candidates = document if isinstance(document, list) else [document]
for candidate in candidates:
if not isinstance(candidate, dict):
continue
rows = candidate.get("Rows")
if isinstance(rows, dict):
return rows
if all(isinstance(key, str) for key in candidate) and candidate:
return candidate
raise MappingUpdateError(f"NTE_Assets table {relative_path} does not contain a Rows object")
def _extract_localization(document: Any, relative_path: str) -> dict[str, Any]:
if not isinstance(document, dict) or not all(isinstance(value, dict) for value in document.values()):
raise MappingUpdateError(f"NTE_Assets localization {relative_path} must contain namespace objects")
return document
def _build_translation_index(localization: dict[str, Any]) -> dict[str, list[tuple[str, str]]]:
result: dict[str, list[tuple[str, str]]] = {}
for namespace, entries in localization.items():
for key, value in entries.items():
if not isinstance(key, str) or not isinstance(value, str):
raise MappingUpdateError(f"invalid English localization entry {namespace}:{key}")
result.setdefault(key, []).append((namespace, value))
return result
def _build_primary_mapping(
rows: dict[str, Any],
kind: str,
translations: dict[str, list[tuple[str, str]]],
) -> dict[str, dict[str, Any]]:
result = {}
for item_id in sorted(rows, key=lambda value: (value.casefold(), value)):
meta = _normalise_row(item_id, rows[item_id], translations)
if kind == "character" and meta["rank"] not in {"S", "A"}:
raise MappingUpdateError(f"unexpected character quality for {item_id}")
result[item_id] = {"name": meta["name"], "rank": meta["rank"]}
return result
def _build_item_mapping(
tables: dict[str, dict[str, Any]],
translations: dict[str, list[tuple[str, str]]],
) -> dict[str, dict[str, Any]]:
inventory = _casefold_index(tables["inventory"], "inventory")
capital = _casefold_index(tables["capital_inventory"], "capital inventory")
appearances = _casefold_index(tables["appearances"], "appearances")
result: dict[str, dict[str, Any]] = {}
for illustrated_id in tables["illustrations"]:
folded = illustrated_id.casefold()
if illustrated_id.isdigit() or folded.startswith("fork_") or folded.startswith("characterawaken_"):
continue
sources = (inventory, capital, appearances)
match = next((source.get(folded) for source in sources if folded in source), None)
if match is None:
raise MappingUpdateError(f"illustrated reward {illustrated_id} is missing from item tables")
canonical_id, row = match
if folded.startswith("fashion_glide_") and folded in appearances:
canonical_id = appearances[folded][0]
meta = _normalise_row(canonical_id, row, translations)
result[canonical_id] = {
"type": "cosmetic" if canonical_id.casefold().startswith("fashion_") else "item",
"name": meta["name"],
"rank": meta["rank"],
}
return dict(sorted(result.items(), key=lambda pair: (pair[0].casefold(), pair[0])))
def _casefold_index(rows: dict[str, Any], label: str) -> dict[str, tuple[str, Any]]:
result: dict[str, tuple[str, Any]] = {}
for item_id, row in rows.items():
folded = item_id.casefold()
if folded in result and result[folded][0] != item_id:
raise MappingUpdateError(f"{label} has case-insensitive duplicate IDs: {result[folded][0]} and {item_id}")
result[folded] = (item_id, row)
return result
def _normalise_row(
item_id: str,
row: Any,
translations: dict[str, list[tuple[str, str]]],
) -> dict[str, str]:
if not isinstance(row, dict):
raise MappingUpdateError(f"NTE_Assets row {item_id} must be an object")
item_name = row.get("ItemName") or row.get("Name")
if not isinstance(item_name, dict):
raise MappingUpdateError(f"NTE_Assets row {item_id} has no ItemName")
name = _translate_name(item_id, item_name, translations)
quality = row.get("ItemQuality") or row.get("Quality")
rank = RANK_BY_QUALITY.get(quality)
if rank is None:
raise MappingUpdateError(f"NTE_Assets row {item_id} has unsupported quality {quality!r}")
return {"name": name.strip(), "rank": rank}
def _translate_name(
item_id: str,
string_reference: dict[str, Any],
translations: dict[str, list[tuple[str, str]]],
) -> str:
key = string_reference.get("Key")
if not isinstance(key, str) or not key:
raise MappingUpdateError(f"NTE_Assets row {item_id} has no localization key")
table_id = string_reference.get("TableId")
expected_namespace = None
if isinstance(table_id, str) and table_id:
expected_namespace = table_id.rsplit("/", 1)[-1].split(".", 1)[0]
matches = translations.get(key, [])
preferred = [value for namespace, value in matches if namespace == expected_namespace]
if len(preferred) == 1:
name = preferred[0]
elif len(matches) == 1:
name = matches[0][1]
elif not matches:
raise MappingUpdateError(f"English localization is missing {item_id} key {key}")
else:
namespaces = ", ".join(namespace for namespace, _value in matches)
raise MappingUpdateError(f"English localization key {key} for {item_id} is ambiguous: {namespaces}")
if not name.strip():
raise MappingUpdateError(f"English localization key {key} for {item_id} is empty")
return name
def _dump_mapping(mapping: dict[str, dict[str, Any]]) -> str:
lines = ["{"]
entries = list(mapping.items())
for index, (item_id, meta) in enumerate(entries):
comma = "," if index < len(entries) - 1 else ""
key = json.dumps(item_id, ensure_ascii=False)
value = json.dumps(meta, ensure_ascii=False, separators=(", ", ": "))
lines.append(f" {key}: {value}{comma}")
lines.append("}")
return "\n".join(lines) + "\n"
def _atomic_write(path: Path, text: str) -> None:
_atomic_write_bytes(path, text.encode("utf-8"))
def _atomic_write_bytes(path: Path, content: bytes) -> None:
with tempfile.NamedTemporaryFile(
mode="wb", dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False
) as handle:
handle.write(content)
temporary = Path(handle.name)
try:
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)

View file

@ -21,3 +21,12 @@ points gifts, chase rewards, batched pages, and bit-packed responses.
Do not replace this file with a real `.pcap`, `.flows`, or exported account
history. Add new cases by constructing the smallest relevant payload, replacing
all timestamps and endpoints, and extending the privacy assertions.
## Synthetic NTE_Assets fixture
`nte_assets/` mirrors only the six table paths and English localization file
consumed by the mapping updater. Its rows, IDs, and names are deliberately
synthetic and contain no account or personal data. The fixture exercises
authoritative additions, metadata updates, deletions, translation lookup,
illustration filtering, and canonical ID casing without depending on the
network or copying a live game-data snapshot into the repository.

View file

@ -0,0 +1 @@
[{"Rows":{"Fashion_Glide_2000":{"Name":{"TableId":"/Game/Text/ST_Appearance.ST_Appearance","Key":"glider_2000","LocalizedString":"Wrong Embedded Appearance"},"Quality":"EItemQuality::ITEM_QUALITY_PURPLE"}}}]

View file

@ -0,0 +1 @@
[{"Rows":{"1003":{"ItemName":{"TableId":"/Game/Text/ST_Player.ST_Player","Key":"character_1003","LocalizedString":"Wrong Embedded Character"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"},"1099":{"ItemName":{"TableId":"/Game/Text/ST_Player.ST_Player","Key":"character_1099","LocalizedString":"Wrong Embedded New Character"},"ItemQuality":"EItemQuality::ITEM_QUALITY_PURPLE"}}}]

View file

@ -0,0 +1 @@
[{"Rows":{"fork_alpha":{"ItemName":{"TableId":"/Game/Text/ST_Fork.ST_Fork","Key":"fork_alpha_name","LocalizedString":"Wrong Embedded Alpha"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"},"fork_new":{"ItemName":{"TableId":"/Game/Text/ST_Fork.ST_Fork","Key":"fork_new_name","LocalizedString":"Wrong Embedded New Arc"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"}}}]

View file

@ -0,0 +1 @@
[{"Rows":{"1003":{},"1099":{},"fork_alpha":{},"fork_new":{},"DIceNormal":{},"Dice_ticket_03":{},"Fashion_glide_2000":{},"Characterawaken_1003":{}}}]

View file

@ -0,0 +1 @@
[{"Rows":{}}]

View file

@ -0,0 +1 @@
[{"Rows":{"DiceNormal":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"dice_normal","LocalizedString":"Wrong Embedded Dice"},"ItemQuality":"EItemQuality::ITEM_QUALITY_ORANGE"},"Dice_ticket_03":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"ticket_03","LocalizedString":"Wrong Embedded Ticket"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"},"Fashion_glide_2000":{"ItemName":{"TableId":"/Game/Text/ST_Appearance.ST_Appearance","Key":"glider_2000","LocalizedString":"Wrong Embedded Glider"},"ItemQuality":"EItemQuality::ITEM_QUALITY_PURPLE"},"UnusedItem":{"ItemName":{"TableId":"/Game/Text/ST_Item.ST_Item","Key":"unused_item","LocalizedString":"Wrong Embedded Unused"},"ItemQuality":"EItemQuality::ITEM_QUALITY_BLUE"}}}]

View file

@ -0,0 +1,18 @@
{
"ST_Player": {
"character_1003": "Old Character Renamed",
"character_1099": "New Character"
},
"ST_Fork": {
"fork_alpha_name": "Alpha Arc",
"fork_new_name": "New Arc"
},
"ST_Item": {
"dice_normal": "Fabricated Dice",
"ticket_03": "New Ticket",
"unused_item": "Not A Pull Reward"
},
"ST_Appearance": {
"glider_2000": "Inventory Glider"
}
}

View file

@ -0,0 +1,184 @@
from __future__ import annotations
import json
import unittest
from copy import deepcopy
from dataclasses import replace
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
from nte_history_exporter.mapping_update import (
MappingUpdateError,
apply_update,
build_mapping_update,
load_assets,
load_current_mappings,
write_update,
)
from tests.support import FIXTURES
SAMPLE_ASSETS = FIXTURES / "nte_assets"
def sample_current():
return {
"arcs.json": {
"fork_alpha": {"name": "Stale Alpha Name", "rank": "B"},
"fork_retired": {"name": "Retired Arc", "rank": "A"},
},
"characters.json": {
"1003": {"name": "Stale Character Name", "rank": "S"},
"1001": {"name": "Retired Character", "rank": "A"},
},
"items.json": {
"DiceNormal": {"type": "item", "name": "Fabricated Dice", "rank": "S"},
"OldTicket": {"type": "item", "name": "Retired Ticket", "rank": "B"},
},
}
def build_sample_update():
current = sample_current()
return current, build_mapping_update(current, load_assets(assets_root=SAMPLE_ASSETS))
class MappingUpdateTests(unittest.TestCase):
def test_mapping_update_is_an_authoritative_snapshot(self):
current, result = build_sample_update()
self.assertEqual(result.mappings["characters.json"]["1003"], {"name": "Old Character Renamed", "rank": "S"})
self.assertEqual(result.mappings["characters.json"]["1099"], {"name": "New Character", "rank": "A"})
self.assertNotIn("1001", result.mappings["characters.json"])
self.assertNotIn("fork_retired", result.mappings["arcs.json"])
self.assertNotIn("OldTicket", result.mappings["items.json"])
self.assertNotEqual(result.mappings["characters.json"]["1003"], current["characters.json"]["1003"])
self.assertEqual(result.report["changes"], {"additions": 4, "updates": 2, "deletions": 3})
self.assertIs(result.report["safety"]["authoritative_snapshot"], True)
self.assertIs(result.report["safety"]["deletions_allowed"], True)
self.assertIs(result.report["safety"]["uid_inputs_touched"], False)
self.assertIs(result.report["safety"]["pool_mappings_touched"], False)
def test_items_use_illustrations_as_filter_and_asset_tables_as_authority(self):
_current, result = build_sample_update()
self.assertEqual(
result.mappings["items.json"]["Fashion_Glide_2000"],
{"type": "cosmetic", "name": "Inventory Glider", "rank": "A"},
)
self.assertIn("DiceNormal", result.mappings["items.json"])
self.assertNotIn("DIceNormal", result.mappings["items.json"])
self.assertNotIn("UnusedItem", result.mappings["items.json"])
self.assertFalse(any(key.startswith("Characterawaken_") for key in result.mappings["items.json"]))
def test_embedded_localized_strings_are_ignored(self):
_current, result = build_sample_update()
self.assertEqual(result.mappings["characters.json"]["1003"]["name"], "Old Character Renamed")
self.assertNotEqual(result.mappings["characters.json"]["1003"]["name"], "Wrong Embedded Character")
def test_mapping_update_rejects_missing_english_translation(self):
assets = load_assets(assets_root=SAMPLE_ASSETS)
tables = deepcopy(assets.tables)
del tables["localization"]["ST_Player"]["character_1003"]
with self.assertRaisesRegex(MappingUpdateError, "English localization is missing"):
build_mapping_update(sample_current(), replace(assets, tables=tables))
def test_mapping_update_rejects_ambiguous_translation_without_namespace_match(self):
assets = load_assets(assets_root=SAMPLE_ASSETS)
tables = deepcopy(assets.tables)
tables["inventory"]["Fashion_glide_2000"]["ItemName"]["TableId"] = "/Game/Text/Unknown.Unknown"
tables["localization"]["OtherNamespace"] = {"glider_2000": "Other Glider"}
with self.assertRaisesRegex(MappingUpdateError, "is ambiguous"):
build_mapping_update(sample_current(), replace(assets, tables=tables))
def test_local_asset_source_has_reproducible_provenance(self):
first = load_assets(assets_root=SAMPLE_ASSETS)
second = load_assets(assets_root=SAMPLE_ASSETS)
self.assertEqual(first.sha256, second.sha256)
self.assertEqual(first.file_sha256, second.file_sha256)
self.assertEqual(first.source, str(SAMPLE_ASSETS.resolve()))
self.assertIsNone(first.source_ref)
def test_mapping_update_writes_deterministic_review_artifacts(self):
_current, result = build_sample_update()
with TemporaryDirectory() as first_tmp, TemporaryDirectory() as second_tmp:
first = Path(first_tmp)
second = Path(second_tmp)
write_update(result, first)
write_update(result, second)
for filename in (*result.mappings, "mapping-update-report.json"):
self.assertEqual((first / filename).read_bytes(), (second / filename).read_bytes())
report = json.loads((first / "mapping-update-report.json").read_text(encoding="utf-8"))
self.assertEqual(report["changes"]["deletions"], 3)
def test_mapping_update_does_not_mutate_inputs(self):
current = sample_current()
original = deepcopy(current)
build_mapping_update(current, load_assets(assets_root=SAMPLE_ASSETS))
self.assertEqual(current, original)
def test_mapping_update_apply_touches_only_reward_mapping_files(self):
_current, result = build_sample_update()
with TemporaryDirectory() as tmp:
directory = Path(tmp)
for filename, mapping in sample_current().items():
(directory / filename).write_text(json.dumps(mapping), encoding="utf-8")
protected = directory / "permanent_board.json"
protected.write_text('{"protected": true}\n', encoding="utf-8")
apply_update(result, directory)
self.assertEqual(load_current_mappings(directory), result.mappings)
self.assertEqual(protected.read_text(encoding="utf-8"), '{"protected": true}\n')
self.assertFalse((directory / "mapping-update-report.json").exists())
def test_mapping_update_apply_rolls_back_partial_write(self):
_current, result = build_sample_update()
with TemporaryDirectory() as tmp:
directory = Path(tmp)
originals = {}
for filename, mapping in sample_current().items():
destination = directory / filename
destination.write_text(json.dumps(mapping), encoding="utf-8")
originals[filename] = destination.read_bytes()
from nte_history_exporter import mapping_update
real_write = mapping_update._atomic_write
calls = 0
def fail_second_write(path, text):
nonlocal calls
calls += 1
if calls == 2:
raise OSError("synthetic write failure")
return real_write(path, text)
with (
patch("nte_history_exporter.mapping_update._atomic_write", side_effect=fail_second_write),
self.assertRaisesRegex(OSError, "synthetic write failure"),
):
apply_update(result, directory)
for filename, original in originals.items():
self.assertEqual((directory / filename).read_bytes(), original)
def test_mapping_update_rejects_unresolved_illustrated_rewards(self):
assets = load_assets(assets_root=SAMPLE_ASSETS)
tables = deepcopy(assets.tables)
tables["illustrations"]["MissingReward"] = {}
with self.assertRaisesRegex(MappingUpdateError, "MissingReward"):
build_mapping_update(sample_current(), replace(assets, tables=tables))
def test_load_assets_rejects_missing_tables(self):
with TemporaryDirectory() as tmp, self.assertRaisesRegex(MappingUpdateError, "cannot read"):
load_assets(assets_root=Path(tmp))

86
tools/update_mappings.py Normal file
View file

@ -0,0 +1,86 @@
from __future__ import annotations
import argparse
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / "src"
if str(SRC) not in sys.path:
sys.path.insert(0, str(SRC))
from nte_history_exporter.mapping_update import ( # noqa: E402
DEFAULT_SOURCE_REF,
MappingUpdateError,
apply_update,
build_mapping_update,
load_assets,
load_current_mappings,
write_update,
)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Synchronise reward mappings from the authoritative NTE_Assets tables."
)
parser.add_argument(
"--assets-root",
type=Path,
help="local NTE_Assets checkout (downloads the required tables when omitted)",
)
parser.add_argument(
"--source-ref",
default=DEFAULT_SOURCE_REF,
help="NTE_Assets branch, tag, or commit to download (default: main)",
)
parser.add_argument(
"--output-dir",
type=Path,
default=ROOT / "build" / "mapping-update",
help="directory for staged mappings and the review report",
)
parser.add_argument(
"--apply",
action="store_true",
help="replace reward mapping files after staging and validation",
)
parser.add_argument(
"--check",
action="store_true",
help="exit 1 when the generated snapshot differs from local mappings",
)
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
assets = load_assets(assets_root=args.assets_root, source_ref=args.source_ref)
current = load_current_mappings(ROOT / "mappings")
result = build_mapping_update(current, assets)
write_update(result, args.output_dir)
if args.apply:
apply_update(result, ROOT / "mappings")
except (MappingUpdateError, OSError) as exc:
print(f"mapping update failed: {exc}", file=sys.stderr)
return 2
changes = result.report["changes"]
print(f"source: {assets.source}")
if assets.source_ref:
print(f"source ref: {assets.source_ref}")
print(f"staged: {args.output_dir}")
print(
"changes: "
f"{changes['additions']} additions, "
f"{changes['updates']} updates, "
f"{changes['deletions']} deletions"
)
if args.apply:
print("applied: mappings/arcs.json, mappings/characters.json, mappings/items.json")
return 1 if args.check and result.change_count else 0
if __name__ == "__main__":
raise SystemExit(main())