From 2188e62d4b5da6c25ca489a89d3e12427b028320 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Tue, 23 Jun 2026 15:14:20 +0100 Subject: [PATCH 1/4] update release workflow to catch forgotten version bumping I forgot to bump the version to 0.1.8 in the last release but my workflows still ran and replaced the 0.1.7 files. Added a validation check and now refuses to release on an existing tag. --- .github/workflows/release.yml | 46 +++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7f9d2b1..c995162 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,9 +7,38 @@ permissions: contents: write jobs: + validate-release: + name: Validate Release + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + + steps: + - name: Checkout Source Code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Check release tag is new + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + version="$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")" + tag="v${version}" + + if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then + echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow." + exit 1 + fi + + if gh release view "$tag" >/dev/null 2>&1; then + echo "::error title=GitHub release already exists::${tag} already has a GitHub release. Bump project.version in pyproject.toml before running the release workflow." + exit 1 + fi + build-windows: name: Build Windows if: github.ref == 'refs/heads/main' + needs: validate-release runs-on: windows-latest steps: @@ -44,6 +73,7 @@ jobs: build-linux: name: Build Linux if: github.ref == 'refs/heads/main' + needs: validate-release runs-on: ubuntu-latest steps: @@ -81,6 +111,7 @@ jobs: build-macos: name: Build Macos if: github.ref == 'refs/heads/main' + needs: validate-release runs-on: macos-15-intel steps: @@ -195,9 +226,14 @@ jobs: nte-history-exporter-v${version}-x86_64-apple-darwin.zip ) - if gh release view "$tag" >/dev/null 2>&1; then - gh release edit "$tag" --notes-file release-notes.md - gh release upload "$tag" "${files[@]}" --clobber - else - gh release create "$tag" "${files[@]}" --title "${tag} Release" --notes-file release-notes.md + if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then + echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow." + exit 1 fi + + if gh release view "$tag" >/dev/null 2>&1; then + echo "::error title=GitHub release already exists::${tag} already has a GitHub release. Bump project.version in pyproject.toml before running the release workflow." + exit 1 + fi + + gh release create "$tag" "${files[@]}" --title "${tag} Release" --notes-file release-notes.md From fe3eb986def103f53795eddb0ea9169f845e9542 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Tue, 23 Jun 2026 15:46:37 +0100 Subject: [PATCH 2/4] all non main releases all non main releases will be marked as a pre release so we can build releases that do not get sent out as an update notification to everyone just yet --- .github/workflows/release.yml | 32 ++++++++++++++++-------- src/nte_history_exporter/update_check.py | 3 +++ tests/test_boundary_export.py | 9 +++++++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c995162..37cd713 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,6 @@ permissions: jobs: validate-release: name: Validate Release - if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest steps: @@ -23,7 +22,12 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | version="$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")" - tag="v${version}" + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + tag="v${version}" + else + safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')" + tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}" + fi if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null; then echo "::error title=Release tag already exists::${tag} already exists. Bump project.version in pyproject.toml before running the release workflow." @@ -37,7 +41,6 @@ jobs: build-windows: name: Build Windows - if: github.ref == 'refs/heads/main' needs: validate-release runs-on: windows-latest @@ -72,7 +75,6 @@ jobs: build-linux: name: Build Linux - if: github.ref == 'refs/heads/main' needs: validate-release runs-on: ubuntu-latest @@ -110,7 +112,6 @@ jobs: build-macos: name: Build Macos - if: github.ref == 'refs/heads/main' needs: validate-release runs-on: macos-15-intel @@ -145,7 +146,6 @@ jobs: release: name: Release - if: github.ref == 'refs/heads/main' needs: [build-windows, build-linux, build-macos] runs-on: ubuntu-latest @@ -164,9 +164,14 @@ jobs: - name: Build Release Notes run: | version="${{ steps.package-version.outputs.version }}" - tag="v${version}" + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + tag="v${version}" + else + safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')" + tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}" + fi repo_url="${{ github.server_url }}/${{ github.repository }}" - previous_tag="$(git tag --merged HEAD --sort=-v:refname | grep -E '^v[0-9]+(\.[0-9]+)*' | grep -vx "$tag" | head -n 1 || true)" + previous_tag="$(git tag --merged HEAD --sort=-v:refname | grep -E '^v[0-9]+(\.[0-9]+)*$' | grep -vx "$tag" | head -n 1 || true)" { echo "Release ${tag}" @@ -216,7 +221,14 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | version="${{ steps.package-version.outputs.version }}" - tag="v${version}" + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + tag="v${version}" + prerelease_args=() + else + safe_ref="$(echo "${{ github.ref_name }}" | tr '/_' '--' | tr -cd '[:alnum:].-')" + tag="v${version}-${safe_ref}.${GITHUB_RUN_NUMBER}" + prerelease_args=(--prerelease) + fi files=( nte-history-exporter.exe nte-history-exporter-linux @@ -236,4 +248,4 @@ jobs: exit 1 fi - gh release create "$tag" "${files[@]}" --title "${tag} Release" --notes-file release-notes.md + gh release create "$tag" "${files[@]}" --title "${tag} Release" --notes-file release-notes.md "${prerelease_args[@]}" diff --git a/src/nte_history_exporter/update_check.py b/src/nte_history_exporter/update_check.py index 53b484f..79f79eb 100644 --- a/src/nte_history_exporter/update_check.py +++ b/src/nte_history_exporter/update_check.py @@ -29,6 +29,9 @@ def check_for_update(current_version: str, *, timeout: float = 1.5) -> UpdateInf except Exception: return None + if latest.get("prerelease"): + return None + latest_version = str(latest.get("tag_name", "")).strip() release_url = str(latest.get("html_url", "")).strip() or RELEASES_URL if not latest_version or not is_newer_version(latest_version, current_version): diff --git a/tests/test_boundary_export.py b/tests/test_boundary_export.py index 9e1c925..4979583 100644 --- a/tests/test_boundary_export.py +++ b/tests/test_boundary_export.py @@ -225,6 +225,15 @@ class BoundaryExportTests(unittest.TestCase): ), ) + def test_update_check_ignores_prerelease(self): + latest = { + "tag_name": "v0.1.8-dev-branch.123", + "html_url": "https://github.com/Golumpa/nte-exporter/releases/tag/v0.1.8-dev-branch.123", + "prerelease": True, + } + with patch("nte_history_exporter.update_check.fetch_latest_release", return_value=latest): + self.assertIsNone(check_for_update("0.1.7", timeout=0.1)) + def test_update_check_is_quiet_when_unavailable_or_current(self): with patch("nte_history_exporter.update_check.fetch_latest_release", side_effect=OSError("offline")): self.assertIsNone(check_for_update("0.1.6", timeout=0.1)) From 241a15e58412f8c74eafdd7d143c559e8e2194e9 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Tue, 23 Jun 2026 15:49:09 +0100 Subject: [PATCH 3/4] update version to 0.1.8 --- docs/export-format.md | 2 +- pyproject.toml | 2 +- src/nte_history_exporter/__init__.py | 2 +- src/nte_history_exporter/constants.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/export-format.md b/docs/export-format.md index 6547903..b10ba29 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -15,7 +15,7 @@ decoder-only offsets. "capture_source": "npcap", "exporter": { "name": "nte-history-exporter", - "version": "0.1.7" + "version": "0.1.8" }, "banner": { "id": "Lottery_Permanent", diff --git a/pyproject.toml b/pyproject.toml index b6a6a90..c98875c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nte-history-exporter" -version = "0.1.7" +version = "0.1.8" description = "Cross-platform Neverness to Everness pull-history exporter." requires-python = ">=3.10" dependencies = [] diff --git a/src/nte_history_exporter/__init__.py b/src/nte_history_exporter/__init__.py index f1380ee..9cb17e7 100644 --- a/src/nte_history_exporter/__init__.py +++ b/src/nte_history_exporter/__init__.py @@ -1 +1 @@ -__version__ = "0.1.7" +__version__ = "0.1.8" diff --git a/src/nte_history_exporter/constants.py b/src/nte_history_exporter/constants.py index 57791c0..c5c35b6 100644 --- a/src/nte_history_exporter/constants.py +++ b/src/nte_history_exporter/constants.py @@ -11,7 +11,7 @@ LIMITED_CHARACTER_BANNER_NAME = "Limited Character Board" ARC_BANNER_ID = "Arc_MiracleBox" ARC_BANNER_NAME = "Arc Miracle Box" EXPORTER_NAME = "nte-history-exporter" -EXPORTER_VERSION = "0.1.7" +EXPORTER_VERSION = "0.1.8" HISTORY_REQUEST_BANNER = 4220 HISTORY_REQUEST_LENGTH = 45 From 98b2165b84b78780f32e98173318427134fd45f9 Mon Sep 17 00:00:00 2001 From: Golumpa Date: Sun, 28 Jun 2026 16:23:02 +0100 Subject: [PATCH 4/4] Stabilize UID generation and page-first dice parsing --- docs/export-format.md | 10 ++-- docs/packet-format.md | 3 ++ src/nte_history_exporter/decoder/arc.py | 6 +-- src/nte_history_exporter/decoder/boundary.py | 3 -- src/nte_history_exporter/decoder/protocol.py | 31 ++++++++++-- tests/test_boundary_export.py | 52 ++++++++++++++++---- 6 files changed, 82 insertions(+), 23 deletions(-) diff --git a/docs/export-format.md b/docs/export-format.md index b10ba29..cac0533 100644 --- a/docs/export-format.md +++ b/docs/export-format.md @@ -92,18 +92,22 @@ nice names or ranks. ## UID generation -The `uid` is the first 32 hex characters of `sha256(source)`. We generate our own roll UID as the game does not send their own, so to make things trackable and to help prevent duplicates we create our own UID with a selection of feilds making each entry in the history 100% unique. The way it is done means on a rescan the same UID is generated for the same history item even if it is further in the history and you have pulled more since the last scan. +The `uid` is the first 32 hex characters of `sha256(source)`. The game does not +appear to send a stable pull/reward row ID, so the exporter builds one from the +history pool, raw packet timestamp, and ordinal inside that timestamp group. +Decoded content such as dice result, reward ID, and quantity is intentionally +excluded so decoder fixes do not change the identity of an already-captured row. Monopoly source: ```text -nte|monopoly|pool_group_id|timestamp_raw|timestamp_group_ordinal|roll_result|reward_key_hex|quantity +nte|monopoly|pool_group_id|timestamp_raw|timestamp_group_ordinal ``` Arc source: ```text -nte|gashapon|pool_group_id|timestamp_raw|timestamp_group_ordinal|reward_key_hex +nte|gashapon|pool_group_id|timestamp_raw|timestamp_group_ordinal ``` ## Example records diff --git a/docs/packet-format.md b/docs/packet-format.md index aa221c6..dacc8de 100644 --- a/docs/packet-format.md +++ b/docs/packet-format.md @@ -25,6 +25,9 @@ Decoded fields: - `roll_result = first u32 / 4` - `roll_result = 0` means Points Gift +- Some page-first records include a one-byte page prefix and an extra `0x14` + field before the real dice u32. In that shape, the real dice u32 is at offset + 9 within the record chunk, not the earlier `0x14` field. - Some page-first records have a short prefix before the record body. For these, a hidden signed source flag immediately after the visible dice field overrides the visible dice: - `source_flag = 0` means Points Gift - `source_flag = -4` means Chase Reward diff --git a/src/nte_history_exporter/decoder/arc.py b/src/nte_history_exporter/decoder/arc.py index baf3400..c353664 100644 --- a/src/nte_history_exporter/decoder/arc.py +++ b/src/nte_history_exporter/decoder/arc.py @@ -146,8 +146,8 @@ def build_arc_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]: return rows -def make_arc_uid(timestamp_raw: str, ordinal: int, arc_key_hex: str) -> str: - source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal), arc_key_hex]) +def make_arc_uid(timestamp_raw: str, ordinal: int) -> str: + source = "|".join([GAME_UID_PART, ARC_SYSTEM, ARC_BANNER_ID, timestamp_raw, str(ordinal)]) return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32] @@ -165,7 +165,7 @@ def annotate_arc_groups(rows: list[dict[str, Any]]) -> None: row["timestamp_group_index"] = group_index row["timestamp_group_ordinal"] = ordinal row["timestamp_group_size_seen"] = len(indexes) - row["uid"] = make_arc_uid(timestamp_raw, ordinal, row["reward_key_hex"]) + row["uid"] = make_arc_uid(timestamp_raw, ordinal) row["uid_status"] = "stable" row["export_record"] = True row["skip_reason"] = "" diff --git a/src/nte_history_exporter/decoder/boundary.py b/src/nte_history_exporter/decoder/boundary.py index 1f2394a..4e893b7 100644 --- a/src/nte_history_exporter/decoder/boundary.py +++ b/src/nte_history_exporter/decoder/boundary.py @@ -14,9 +14,6 @@ def make_uid(record: dict[str, Any], ordinal: int) -> str: str(record.get("pool_group_id", BANNER_ID)), str(record.get("timestamp_raw_hex", "")), str(ordinal), - str(record.get("dice", "")), - str(record.get("reward_key_hex", "")), - str(record.get("quantity", "")), ] ) return hashlib.sha256(source.encode("utf-8")).hexdigest()[:32] diff --git a/src/nte_history_exporter/decoder/protocol.py b/src/nte_history_exporter/decoder/protocol.py index 0b799ca..6a40b56 100644 --- a/src/nte_history_exporter/decoder/protocol.py +++ b/src/nte_history_exporter/decoder/protocol.py @@ -132,7 +132,20 @@ def extract_key(chunk_without_marker: bytes) -> str: return "" if best is None else chunk_without_marker[best:].hex() +def _page_first_prefixed_dice_raw(chunk_without_marker: bytes) -> int | None: + if len(chunk_without_marker) >= 17 and chunk_without_marker[0] == 0: + prefix_field = struct.unpack_from(" tuple[int | None, int | None, int | None]: + prefixed_dice_raw = _page_first_prefixed_dice_raw(chunk_without_marker) + if prefixed_dice_raw is not None: + return (0 if prefixed_dice_raw == 0 else prefixed_dice_raw // 4), prefixed_dice_raw, 9 + for off in range(0, min(16, max(0, len(chunk_without_marker) - 3))): val = struct.unpack_from(" list[dict[str, chunk = response_content[prev:marker_offset] full_record = response_content[prev : marker_offset + len(marker) + 8] dice, dice_raw, dice_offset = extract_dice(chunk) - if dice is None and len(chunk) > 32: + original_key = extract_key(chunk) + original_key_bytes = bytes.fromhex(original_key) if original_key else b"" + key_position = chunk.find(original_key_bytes) if original_key_bytes else len(chunk) + should_try_embedded_trim = ( + dice is None + or (_page_first_prefixed_dice_raw(chunk) is None and key_position > 32) + ) + if should_try_embedded_trim and len(chunk) > 32: embedded_candidates = [] - original_key = extract_key(chunk) - original_key_bytes = bytes.fromhex(original_key) if original_key else b"" for trim in range(1, min(96, len(chunk))): candidate = chunk[trim:] candidate_dice, candidate_raw, candidate_offset = extract_dice(candidate) + candidate_is_page_first = _page_first_prefixed_dice_raw(candidate) is not None if ( candidate_dice is not None - and candidate_offset in (0, 5) + and candidate_offset in (0, 5, 9) + and (dice is None or candidate_is_page_first) and extract_key(candidate) == original_key ): key_count = candidate.count(original_key_bytes) if original_key_bytes else 0 @@ -218,6 +238,7 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str, embedded_candidates.append( ( -key_count, + not candidate_is_page_first, candidate_offset != 5, key_position, -trim, @@ -229,7 +250,7 @@ def _decode_aligned_response_records(response_content: bytes) -> list[dict[str, ) ) if embedded_candidates: - _, _, _, _, trim, chunk, dice, dice_raw, dice_offset = min(embedded_candidates) + _, _, _, _, _, trim, chunk, dice, dice_raw, dice_offset = min(embedded_candidates) record_start = prev + trim full_record = response_content[prev + trim : marker_offset + len(marker) + 8] key_hex = extract_key(chunk) diff --git a/tests/test_boundary_export.py b/tests/test_boundary_export.py index 4979583..7d82cc9 100644 --- a/tests/test_boundary_export.py +++ b/tests/test_boundary_export.py @@ -300,9 +300,9 @@ class BoundaryExportTests(unittest.TestCase): def test_uid_source_matches_v4_reference(self): rows = load_reference_csv("monopoly_history_poc_10_all_44_pages_v4.csv") first = rows[0] - self.assertEqual(make_uid(first, 0), "5adcf52282e15445466863b271f3b745") + self.assertEqual(make_uid(first, 0), "f2c72f0a80b79216bf15661521620693") - def test_uid_uses_detected_pool_group_id(self): + def test_uid_uses_pool_timestamp_and_ordinal_only(self): row = { "pool_group_id": "Lottery_LimitedCharacter", "timestamp_raw_hex": "40e93247c3097b23", @@ -310,7 +310,18 @@ class BoundaryExportTests(unittest.TestCase): "reward_key_hex": "10a58d957dd1a58dad95d17dc1c800", "quantity": 50, } - self.assertNotEqual(make_uid(row, 0), "5adcf52282e15445466863b271f3b745") + changed_content = { + **row, + "dice": 1, + "reward_key_hex": "98bdc9ad7dd9a5b99501", + "quantity": 1, + } + changed_pool = {**row, "pool_group_id": "Lottery_Permanent"} + + self.assertEqual(make_uid(row, 0), "74a9ef4aacde549dfe8e8e7cc6ddd65b") + self.assertEqual(make_uid(changed_content, 0), make_uid(row, 0)) + self.assertNotEqual(make_uid(changed_pool, 0), make_uid(row, 0)) + self.assertNotEqual(make_uid(row, 1), make_uid(row, 0)) def test_pages_1_to_5_exports_every_row(self): rows = load_reference_csv("monopoly_history_poc_13_pages_1_to_5_v4.csv") @@ -592,7 +603,7 @@ class BoundaryExportTests(unittest.TestCase): self.assertEqual(decoded["dice_raw_u32"], -4) self.assertEqual(decoded["reward_id"], "Dice_ticket_01") self.assertEqual(decoded["quantity"], 30) - self.assertEqual(make_uid(decoded, int(reference["timestamp_group_ordinal"])), "7d035ec098f856f81b403ea538810145") + self.assertEqual(make_uid(decoded, int(reference["timestamp_group_ordinal"])), "23dc293f39f18e94a81ccaaf7e1a67eb") def test_warp_piece_chase_subrecord_without_prefix_marker_is_chase_reward(self): decoded = decode_single_record( @@ -608,6 +619,32 @@ class BoundaryExportTests(unittest.TestCase): self.assertEqual(decoded["reward_name"], "Warp Piece") self.assertEqual(decoded["quantity"], 30) + def test_page_first_prefix_uses_real_dice_field(self): + cases = [ + ( + "003006000014000000040000002800000098bdc9ad7dd9a5b995010000000008000000" + "3c00000010a58d957dd1a58dad95d17dc1c4002800000098bdc9ad7dd9a5b995014c" + "0000000c85c99141bdbdb17d0da185c9858dd195c901c0dd53bd2b137b23", + 1, + "fork_vine", + ), + ( + "00c8060000140000001000000014000000c4c0d4d400000000000400000014000000" + "c4c0d4d400440000000c85c99141bdbdb17d3995dd49bdb1950100d929e115087b23", + 4, + "1055", + ), + ] + for record_hex, expected_dice, reward_id in cases: + with self.subTest(reward_id=reward_id): + decoded = decode_single_record(record_hex) + + self.assertEqual(decoded["dice"], expected_dice) + self.assertEqual(decoded["dice_raw_u32"], expected_dice * 4) + self.assertEqual(decoded["dice_offset_in_record"], 9) + self.assertEqual(decoded["result_type"], "dice") + self.assertEqual(decoded["reward_id"], reward_id) + def test_batched_monopoly_response_normalizes_embedded_page_header(self): page_7 = [load_v7_row("limited_all_04_v7.csv", row) for row in range(31, 36)] page_8 = [load_v7_row("limited_all_04_v7.csv", row) for row in range(36, 41)] @@ -626,7 +663,7 @@ class BoundaryExportTests(unittest.TestCase): self.assertEqual(len(decoded), 10) self.assertEqual(decoded[5]["reward_id"], page_8[0]["reward_id"]) - self.assertEqual(decoded[5]["dice"], 5) + self.assertEqual(decoded[5]["dice"], 4) self.assertEqual(decoded[5]["result_type"], "dice") self.assertEqual(decoded[5]["record_hex"], page_8[0]["record_hex"]) @@ -665,10 +702,7 @@ class BoundaryExportTests(unittest.TestCase): self.assertEqual(decode_arc_key(bytes.fromhex(row["arc_key_hex"])), "fork_nonos") _ticks, _unix, decoded = decode_arc_timestamp(bytes.fromhex(row["timestamp_raw_hex"])) self.assertEqual(decoded, "2026-06-10 23:46:29") - self.assertEqual( - make_arc_uid(row["timestamp_raw_hex"], int(row["timestamp_group_ordinal"]), row["arc_key_hex"]), - row["uid"], - ) + self.assertEqual(make_arc_uid(row["timestamp_raw_hex"], int(row["timestamp_group_ordinal"])), "4435d9729fa8fd0eaf1b1ad7aa4d2172") def test_arc_response_parser_matches_reference_first_page(self): reference_rows = load_arc_csv("arc_pull_10_all_pages_v2.csv")[:5]