goober/nte-exporter
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add safe snapshot and segment assembly

Browse source 
Add envelope-aware snapshot and segment assembly for structured Monopoly and Arc history records.
This commit is contained in:
Golumpa 2026-07-15 19:48:11 +01:00
parent a5312520e7
commit 73bb100cd6
8 changed files with 601 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

52
src/nte_history_exporter/decoder/run.py
View file

@ -4,7 +4,11 @@ from datetime import datetime, timezone
from typing import Any
from nte_history_exporter.constants import POOL_META
from nte_history_exporter.decoder.protocol import decode_response_records
from nte_history_exporter.decoder.protocol import decode_response_records, structured_monopoly_rows
from nte_history_exporter.decoder.structured_protocol import (
StructuredProtocolAssembler,
parse_structured_blocks,
)
def fmt_packet_time(ts: float | None) -> str:
@ -13,7 +17,7 @@ def fmt_packet_time(ts: float | None) -> str:
return datetime.fromtimestamp(ts, timezone.utc).strftime("%H:%M:%S.%f")[:-3]
def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
def _build_primary_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
rows_out: list[dict[str, Any]] = []
for pair in pairs:
page, offset, req_i, req_ts, resp_i, resp_ts, response_content = pair[:7]
@ -59,3 +63,47 @@ def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
}
)
return rows_out
def build_rows_from_pairs(pairs: list[tuple]) -> list[dict[str, Any]]:
primary_rows = _build_primary_rows_from_pairs(pairs)
decoded_rows = [row for row in primary_rows if row.get("record_count", 0)]
if decoded_rows and any(row.get("decoder_mode") != "structured_fallback" for row in decoded_rows):
return primary_rows
assembler = StructuredProtocolAssembler()
for source_index, pair in enumerate(pairs):
assembler.add_blocks(
parse_structured_blocks(pair[6], "monopoly", source_index=source_index)
)
assembled = assembler.rows("monopoly")
if not assembled:
return primary_rows
rows_out: list[dict[str, Any]] = []
converted = structured_monopoly_rows(assembled)
for row_index, (record, structured) in enumerate(zip(converted, assembled), start=1):
source_index = structured.source_index or 0
pair = pairs[source_index]
page, offset, req_i, req_ts, resp_i, resp_ts, response_content = pair[:7]
kind = pair[7] if len(pair) > 7 else "permanent"
pool = POOL_META.get(kind, POOL_META["permanent"])
rows_out.append(
{
"page": page,
"offset": offset,
"pool_group_id": pool["id"],
"pool_group_name": pool["name"],
"request_msg": req_i,
"request_time_utc": fmt_packet_time(req_ts),
"response_msg": resp_i,
"response_time_utc": fmt_packet_time(resp_ts),
"response_len": len(response_content),
"record_count": len(assembled),
**record,
"row": row_index,
"structured_assembly": "snapshot_segments",
"structured_assembly_warning_count": len(assembler.warnings),
}
)
return rows_out