From 01dd30516883fe601ef92ee458c9cbb893347b55 Mon Sep 17 00:00:00 2001 From: Vriff <74007411+vrifftech@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:15:52 -0400 Subject: [PATCH 1/2] Fixing reported issues with embedded txi data in tpcs and list reordering --- .../pykotor/resource/formats/tpc/io_tpc.py | 159 +++++++++++++++--- .../pykotor/resource/formats/tpc/tpc_data.py | 30 +++- .../pykotor/resource/formats/txi/io_txi.py | 34 +--- .../pykotor/resource/formats/txi/txi_data.py | 16 +- .../src/pykotor/resource/generics/utc.py | 90 +++++++--- .../tests/resource/formats/test_tpc.py | 100 ++++++++++- .../tests/resource/formats/test_txi_data.py | 43 +++++ .../tests/resource/generics/test_utc.py | 72 ++++++++ 8 files changed, 460 insertions(+), 84 deletions(-) diff --git a/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py b/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py index f96a7b7c1..20c9f7df5 100644 --- a/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py +++ b/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py @@ -108,6 +108,37 @@ class TPCBinaryReader(ResourceReader): MAX_DIMENSIONS: Literal[0x8000] = 0x8000 IMG_DATA_START_OFFSET: Literal[0x80] = 0x80 + TXI_COMMAND_HINTS = frozenset( + { + "alphamean", + "blending", + "bumpmapscaling", + "bumpmaptexture", + "bumpyshinytexture", + "clamp", + "compresstexture", + "cube", + "decal", + "defaultheight", + "defaultwidth", + "distort", + "distortionamplitude", + "downsamplemax", + "downsamplemin", + "envmaptexture", + "fps", + "isbumpmap", + "islightmap", + "mipmap", + "numx", + "numy", + "proceduretype", + "speed", + "waterheight", + "waterwidth", + "xbox_downsample", + } + ) def __init__( self, @@ -120,9 +151,13 @@ def __init__( @autoclose def load(self, *, auto_close: bool = True) -> TPC: # noqa: PLR0912, C901, PLR0915 data = self._reader.read_all() + # The Kaitai parser is used as a best-effort structural smoke test only. + # PyKotor's manual reader below is authoritative because retail TPCs, + # especially animated textures with embedded TXI footers, use header + # combinations that generated parsers have historically misinterpreted. try: Tpc.from_bytes(data) - except kaitaistruct.KaitaiStructError: + except (kaitaistruct.KaitaiStructError, UnicodeDecodeError, OSError, ValueError): pass self._reader = BinaryReader.from_bytes(data, 0) @@ -130,8 +165,9 @@ def load(self, *, auto_close: bool = True) -> TPC: # noqa: PLR0912, C901, PLR09 self._layer_count: int = 1 self._mipmap_count: int = 0 - data_size: int = self._reader.read_uint32() # 0x0 - compressed: bool = data_size != 0 + header_data_size: int = self._reader.read_uint32() # 0x0 + data_size: int = header_data_size + compressed: bool = header_data_size != 0 alpha_test: float = self._reader.read_single() # 0x4 self._tpc.alpha_test = alpha_test width: int = self._reader.read_uint16() # 0x8 @@ -179,15 +215,22 @@ def load(self, *, auto_close: bool = True) -> TPC: # noqa: PLR0912, C901, PLR09 complete_data_size += tpc_format.get_size(reduced_width, reduced_height) complete_data_size *= self._layer_count + texture_data_size = self._select_texture_data_size( + data, + default_size=complete_data_size, + declared_size=header_data_size, + ) - self._reader.skip(0x72 + complete_data_size) + self._reader.seek(min(self.IMG_DATA_START_OFFSET + texture_data_size, self._reader.size())) txi_data_size: int = self._reader.size() - self._reader.position() if txi_data_size > 0: - self._tpc.txi = self._reader.read_string( + txi_text = self._reader.read_string( txi_data_size, encoding="ascii", errors="ignore", - ) + ).split("\x00", maxsplit=1)[0] + if txi_text.strip(): + self._tpc.txi = txi_text txi_data: TXI = self._tpc._txi # noqa: SLF001 self._tpc.is_animated = bool( @@ -218,10 +261,11 @@ def load(self, *, auto_close: bool = True) -> TPC: # noqa: PLR0912, C901, PLR09 if compressed and not self._tpc.is_animated: expected_size: int = tpc_format.get_size(width, height) - if data_size != expected_size: + if data_size < expected_size: raise ValueError( f"Invalid data size for a texture of {width}x{height} pixels and format {tpc_format!r}" ) + data_size = expected_size self._reader.seek(self.IMG_DATA_START_OFFSET) if ( @@ -236,7 +280,7 @@ def load(self, *, auto_close: bool = True) -> TPC: # noqa: PLR0912, C901, PLR09 width, height, ) - full_data_size: int = self._reader.size() - self.IMG_DATA_START_OFFSET + full_data_size: int = texture_data_size if full_data_size < (self._layer_count * full_image_data_size): msg: str = f"Insufficient data for image. Expected at least {hex(self._layer_count * full_image_data_size)} bytes, but only {hex(full_data_size)} bytes are available." raise ValueError( @@ -301,6 +345,82 @@ def load(self, *, auto_close: bool = True) -> TPC: # noqa: PLR0912, C901, PLR09 return self._tpc + def _select_texture_data_size( + self, + data: bytes, + *, + default_size: int, + declared_size: int, + ) -> int: + """Choose the texture payload length before an embedded TXI footer. + + Some TPC writers store only the first mip level size in the compressed + ``data_size`` header field, while others store the full texture payload + size. Prefer offsets that actually look like an ASCII TXI footer; fall + back to PyKotor's historical size calculation otherwise. + """ + candidates: list[int] = [] + if declared_size > 0: + candidates.append(declared_size) + if self._layer_count > 1: + candidates.append(declared_size * self._layer_count) + candidates.append(default_size) + + seen: set[int] = set() + for size in candidates: + if size in seen: + continue + seen.add(size) + txi_offset = self.IMG_DATA_START_OFFSET + size + if self._looks_like_txi_at(data, txi_offset): + return size + + shifted_size = self._find_nearby_txi_size(data, size) + if shifted_size is not None: + return shifted_size + return default_size + + def _find_nearby_txi_size(self, data: bytes, size: int) -> int | None: + # Legacy animated output can undercount the packed frame/mip payload by + # a few DXT min-size blocks because the header mip count was written for + # per-frame mips before the TXI footer had been parsed. + txi_offset = self.IMG_DATA_START_OFFSET + size + scan_end = min(len(data), txi_offset + 1024) + for offset in range(txi_offset + 1, scan_end): + if self._looks_like_txi_at(data, offset): + return offset - self.IMG_DATA_START_OFFSET + return None + + @classmethod + def _looks_like_txi_at(cls, data: bytes, offset: int) -> bool: + if offset < cls.IMG_DATA_START_OFFSET or offset >= len(data): + return False + + footer = data[offset:] + raw = footer.split(b"\x00", maxsplit=1)[0].lstrip(b"\r\n\t ") + if not raw: + return False + + first_byte = raw[0] + if not (65 <= first_byte <= 90 or 97 <= first_byte <= 122): + return False + + sample = raw[:4096] + if any(byte < 32 and byte not in b"\r\n\t" for byte in sample): + return False + + text = raw.decode("ascii", errors="ignore").strip() + if not text: + return False + + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + command = stripped.split(None, maxsplit=1)[0].lower() + return command in cls.TXI_COMMAND_HINTS + return False + def _normalize_cubemaps(self): self._tpc.convert( TPCTextureFormat.RGB @@ -427,16 +547,17 @@ def write(self, *, auto_close: bool = True): # noqa: FBT001, FBT002, ARG002 # raise ValueError(f"Cubemap must have exactly 6 layers, found {self._layer_count}") height = frame_height * self._layer_count - # Calculate data size - base_level_size: int = ( - len(self._tpc.layers[0].mipmaps[0].data) - if self._tpc.layers and self._tpc.layers[0].mipmaps - else 0 - ) + # Calculate data size. For compressed TPCs this header field is the + # texture payload length before any embedded TXI footer. data_size: int = 0 if tpc_format.is_dxt(): - layers = self._layer_count if self._tpc.is_animated else 1 - data_size = base_level_size * max(1, layers) + data_size = sum( + len(layer.mipmaps[mipmap_idx].data) + for layer in self._tpc.layers + for mipmap_idx in range(min(self._mipmap_count, len(layer.mipmaps))) + ) + + header_mipmap_count: int = 1 if self._tpc.is_animated else self._mipmap_count # Write header (128 bytes) pixel_encoding: int = self._get_pixel_encoding(tpc_format) @@ -445,7 +566,7 @@ def write(self, *, auto_close: bool = True): # noqa: FBT001, FBT002, ARG002 # self._writer.write_uint16(width) # 0x08-0x09: Width self._writer.write_uint16(height) # 0x0A-0x0B: Height self._writer.write_uint8(pixel_encoding) # 0x0C: Pixel encoding - self._writer.write_uint8(self._mipmap_count) # 0x0D: Mipmap count + self._writer.write_uint8(header_mipmap_count) # 0x0D: Mipmap count self._writer.write_bytes(bytes(0x72)) # 0x0E-0x7F: Reserved padding # Write texture data for each layer @@ -485,6 +606,4 @@ def write(self, *, auto_close: bool = True): # noqa: FBT001, FBT002, ARG002 # return txi_lines = txi_payload.split("\n") normalized = "\r\n".join(txi_lines) - if not normalized.endswith("\r\n"): - normalized += "\r\n" - self._writer.write_bytes(normalized.encode("ascii", errors="ignore") + b"\x00") + self._writer.write_bytes(normalized.encode("ascii", errors="ignore")) diff --git a/Libraries/PyKotor/src/pykotor/resource/formats/tpc/tpc_data.py b/Libraries/PyKotor/src/pykotor/resource/formats/tpc/tpc_data.py index b7f77d801..4f6476fcf 100644 --- a/Libraries/PyKotor/src/pykotor/resource/formats/tpc/tpc_data.py +++ b/Libraries/PyKotor/src/pykotor/resource/formats/tpc/tpc_data.py @@ -13,6 +13,7 @@ from __future__ import annotations +from copy import deepcopy from dataclasses import dataclass, field from enum import IntEnum, auto from typing import TYPE_CHECKING @@ -553,6 +554,8 @@ class TPC(BiowareResource): def __init__(self): self._txi: TXI = TXI() + self._txi_text: str = "" + self._txi_source_features = None self._format: TPCTextureFormat = TPCTextureFormat.Invalid self.layers: list[TPCLayer] = [] self.is_animated: bool = False @@ -569,13 +572,30 @@ def from_blank(cls) -> Self: @property def txi(self) -> str: - """Get the TXI data as a string.""" + """Get the TXI data as text. + + Preserve the original TXI footer text when this TPC was loaded from a + source file. The parsed :class:`TXI` object is still populated for + feature inspection, animation detection, and compatibility with callers + that consume ``_txi.features``. + """ + if ( + self._txi_text.strip() + and self._txi_source_features is not None + and vars(self._txi.features) == vars(self._txi_source_features) + ): + return self._txi_text return str(self._txi) @txi.setter def txi(self, value: str): - """Set the TXI data from a string.""" - self._txi.load(value) + """Set the TXI data from a string while retaining the source text.""" + self._txi_text = value.split("\x00", maxsplit=1)[0] if value else "" + self._txi = TXI() + self._txi_source_features = None + if self._txi_text.strip(): + self._txi.load(self._txi_text) + self._txi_source_features = deepcopy(self._txi.features) def format(self) -> TPCTextureFormat: """Get the texture format.""" @@ -720,5 +740,7 @@ def copy(self) -> Self: instance._format = self._format instance.is_animated = self.is_animated instance.is_cube_map = self.is_cube_map - instance._txi = self._txi + instance._txi_text = self._txi_text + instance._txi_source_features = deepcopy(self._txi_source_features) + instance._txi = TXI(self.txi) return instance diff --git a/Libraries/PyKotor/src/pykotor/resource/formats/txi/io_txi.py b/Libraries/PyKotor/src/pykotor/resource/formats/txi/io_txi.py index 691d1267a..21af78e55 100644 --- a/Libraries/PyKotor/src/pykotor/resource/formats/txi/io_txi.py +++ b/Libraries/PyKotor/src/pykotor/resource/formats/txi/io_txi.py @@ -57,6 +57,7 @@ def load(self, *func_args, auto_close: bool = True, **func_kwargs) -> TXI: # no txi_text = Txi.from_bytes(txi_bytes).content except (kaitaistruct.KaitaiStructError, UnicodeDecodeError): txi_text = txi_bytes.decode("ascii", errors="ignore") + txi_text = txi_text.replace("\x00", "") for line in txi_text.splitlines(): try: @@ -344,35 +345,4 @@ def __init__(self, txi: TXI, target: TARGET_TYPES): @autoclose def write(self, *, auto_close: bool = True): # noqa: FBT001, FBT002, ARG002 # pyright: ignore[reportUnusedParameters] - from pykotor.resource.formats.txi.txi_data import TXICommand - - lines: list[str] = [] - for attr, value in vars(self._txi.features).items(): - if value is None: - continue - if isinstance(value, list) and len(value) == 0: - continue - if isinstance(value, str) and len(value) == 0: - continue - upper_attr = attr.upper() - if upper_attr not in TXICommand.__members__: - RobustLogger().error(f"Invalid TXI attribute '{attr}'") - continue - command = TXICommand[upper_attr] - if isinstance(value, bool): - lines.append(f"{command.value} {int(value)}") - elif isinstance(value, (int, float)): - lines.append(f"{command.value} {value}") - elif isinstance(value, list): - if attr in [ - TXICommand.UPPERLEFTCOORDS.value.lower(), - TXICommand.LOWERRIGHTCOORDS.value.lower(), - ]: - lines.append(f"{command.value} {len(value)}") - lines.extend(" ".join(map(str, coord)) for coord in value) - else: - lines.append(f"{command.value} {' '.join(map(str, value))}") - else: - lines.append(f"{command.value} {value}") - txi_string = "\n".join(lines) - self._writer.write_string(txi_string) + self._writer.write_string(str(self._txi)) diff --git a/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py b/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py index 2366aca87..35950a570 100644 --- a/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py +++ b/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py @@ -53,13 +53,15 @@ class TXI(BiowareResource): def __init__(self, txi: str | None = None): self.features: TXIFeatures = TXIFeatures() self._empty: bool = True - if txi and txi.strip(): + if txi is not None: self.load(txi) def load(self, txi: str): # noqa: C901, PLR0912, PLR0915 from pykotor.resource.formats.txi.io_txi import TXIReaderMode + self.features = TXIFeatures() self._empty = True + txi = txi.replace("\x00", "") mode = TXIReaderMode.NORMAL cur_coords: int = 0 max_coords: int = 0 @@ -332,9 +334,9 @@ def get_features(self) -> TXIFeatures: @staticmethod def parse_blending(s: str) -> int: s_norm = (s or "").strip().lower() - if s_norm == "additive": + if s_norm in {"additive", "1"}: return 1 - if s_norm in {"punchthrough", "punch-through"}: + if s_norm in {"punchthrough", "punch-through", "2"}: return 2 return 0 @@ -348,7 +350,9 @@ def __str__(self) -> str: RobustLogger().error(f"Invalid TXI attribute '{attr}'") continue command: TXICommand = TXICommand[upper_attr] - if isinstance(value, bool): + if command == TXICommand.BLENDING and isinstance(value, int): + lines.append(f"{command.value} {self._stringify_blending(value)}") + elif isinstance(value, bool): lines.append(f"{command.value} {int(value)}") elif isinstance(value, (int, float)): lines.append(f"{command.value} {value}") @@ -362,6 +366,10 @@ def __str__(self) -> str: lines.append(f"{command.value} {value}") return "\n".join(lines) + @staticmethod + def _stringify_blending(value: int) -> str: + return {1: "additive", 2: "punchthrough"}.get(value, str(value)) + class TXIFeatures(ComparableMixin): """Stores texture features parsed from TXI file. diff --git a/Libraries/PyKotor/src/pykotor/resource/generics/utc.py b/Libraries/PyKotor/src/pykotor/resource/generics/utc.py index ea99e720f..afbb4e7bf 100644 --- a/Libraries/PyKotor/src/pykotor/resource/generics/utc.py +++ b/Libraries/PyKotor/src/pykotor/resource/generics/utc.py @@ -30,10 +30,52 @@ from pykotor.resource.type import ResourceType if TYPE_CHECKING: - from pykotor.resource.formats.gff import GFFStruct from pykotor.resource.type import SOURCE_TYPES, TARGET_TYPES +def _order_ids_preserving_original_slots( + current_ids: list[int], + original_mapping: dict[int, int], +) -> list[int]: + """Keep original list indices stable and fill removed slots with newly-added ids. + + UTC feat/power widgets represent membership, not list order. When the UI returns checked + values in display order, a plain sort-by-original-index appends new values after all + retained values. That shifts every original entry after a removed item and produces noisy + TSLPatcher diffs. + """ + if not original_mapping: + return list(current_ids) + + current_set: set[int] = set(current_ids) + added_ids: list[int] = [value for value in current_ids if value not in original_mapping] + added_iter = iter(added_ids) + + ordered: list[int] = [] + for original_id, _original_index in sorted(original_mapping.items(), key=lambda item: item[1]): + if original_id in current_set: + ordered.append(original_id) + continue + + replacement = next(added_iter, None) + if replacement is not None: + ordered.append(replacement) + + ordered.extend(added_iter) + return ordered + + +def _has_original_or_nondefault_root_value( + utc: UTC, + label: str, + value: object, + default: object, +) -> bool: + """Return whether a default-valued optional root field should be materialized.""" + original_fields = utc._original_root_field_names + return original_fields is None or label in original_fields or value != default + + class UTC: """Stores creature data. @@ -202,8 +244,9 @@ class UTC: BINARY_TYPE = ResourceType.UTC def __init__(self): - # internal use only, to preserve the original order: + # Internal use only, to preserve round-trip and diff compatibility. self._original_feat_mapping: dict[int, int] = {} + self._original_root_field_names: set[str] | None = None self._extra_unimplemented_skills: list[int] = [] self.resref: ResRef = ResRef.from_blank() @@ -359,6 +402,7 @@ def construct_utc( utc = UTC() root = gff.root + utc._original_root_field_names = set(root._fields) # Root identity: empty strings / blank ResRefs when absent. utc.resref = root.acquire("TemplateResRef", ResRef.from_blank()) @@ -639,12 +683,20 @@ def dismantle_utc( root.set_uint16("PortraitId", utc.portrait_id) # TODO(th3w1zard1): Add these seemingly missing fields into UTCEditor? - root.set_resref("Portrait", utc.portrait_resref) - root.set_uint8("SaveWill", utc.save_will) - root.set_uint8("SaveFortitude", utc.save_fortitude) - root.set_uint8("Morale", utc.morale) - root.set_uint8("MoraleRecovery", utc.morale_recovery) - root.set_uint8("MoraleBreakpoint", utc.morale_breakpoint) + if _has_original_or_nondefault_root_value( + utc, "Portrait", utc.portrait_resref, ResRef.from_blank() + ): + root.set_resref("Portrait", utc.portrait_resref) + if _has_original_or_nondefault_root_value(utc, "SaveWill", utc.save_will, 0): + root.set_uint8("SaveWill", utc.save_will) + if _has_original_or_nondefault_root_value(utc, "SaveFortitude", utc.save_fortitude, 0): + root.set_uint8("SaveFortitude", utc.save_fortitude) + if _has_original_or_nondefault_root_value(utc, "Morale", utc.morale, 0): + root.set_uint8("Morale", utc.morale) + if _has_original_or_nondefault_root_value(utc, "MoraleRecovery", utc.morale_recovery, 0): + root.set_uint8("MoraleRecovery", utc.morale_recovery) + if _has_original_or_nondefault_root_value(utc, "MoraleBreakpoint", utc.morale_breakpoint, 0): + root.set_uint8("MoraleBreakpoint", utc.morale_breakpoint) root.set_uint8("BodyVariation", utc.body_variation) root.set_uint8("TextureVar", utc.texture_variation) @@ -712,29 +764,21 @@ def dismantle_utc( class_struct.set_int32("Class", utc_class.class_id) class_struct.set_int16("ClassLevel", utc_class.class_level) power_list: GFFList = class_struct.set_list("KnownList0", GFFList()) - for power in utc_class.powers: + ordered_powers = _order_ids_preserving_original_slots( + utc_class.powers, + utc_class._original_powers_mapping, + ) + for power in ordered_powers: power_struct = power_list.add(3) power_struct.set_uint16("Spell", power) power_struct.set_uint8("SpellFlags", 1) power_struct.set_uint8("SpellMetaMagic", 0) - def _sort_powers(power_struct: GFFStruct): - return utc_class._original_powers_mapping.get( - power_struct.get_uint16("Spell"), float("inf") - ) - - power_list._structs = sorted(power_list._structs, key=_sort_powers) - feat_list: GFFList = root.set_list("FeatList", GFFList()) - for feat in utc.feats: + ordered_feats = _order_ids_preserving_original_slots(utc.feats, utc._original_feat_mapping) + for feat in ordered_feats: feat_list.add(1).set_uint16("Feat", feat) - # Sort utc.feats according to their original index, stored in utc._original_feat_mapping - def _sort_feats(feat_struct: GFFStruct): - return utc._original_feat_mapping.get(feat_struct.get_uint16("Feat"), float("inf")) - - feat_list._structs = sorted(feat_list._structs, key=_sort_feats) - # Not sure what these are for, verified they exist in K1's 'c_drdg.utc' in data\templates.bif. Might be unused in which case this can be deleted. if utc._extra_unimplemented_skills: for val in utc._extra_unimplemented_skills: diff --git a/Libraries/PyKotor/tests/resource/formats/test_tpc.py b/Libraries/PyKotor/tests/resource/formats/test_tpc.py index c8a1169e6..c3da39101 100644 --- a/Libraries/PyKotor/tests/resource/formats/test_tpc.py +++ b/Libraries/PyKotor/tests/resource/formats/test_tpc.py @@ -22,7 +22,7 @@ ndix_compressor_available, ) from pykotor.resource.formats.tpc.manipulate.mipmap_ndix import _js_round, downsample_rgba_ndix -from pykotor.resource.formats.tpc.tpc_auto import bytes_tpc +from pykotor.resource.formats.tpc.tpc_auto import bytes_tpc, read_tpc from pykotor.resource.formats.tpc.tpc_auto import build_tpc_from_tga_bytes from pykotor.resource.type import ResourceType from pykotor.resource.formats.tpc.tpc_data import TPC, TPCLayer, TPCMipmap, TPCTextureFormat @@ -106,6 +106,52 @@ def _minimal_uncompressed_tga(width: int, height: int, pixel_depth: int) -> byte raise ValueError(pixel_depth) +def _dxt_mip_chain(width: int, height: int, tpc_format: TPCTextureFormat, seed: int) -> list[TPCMipmap]: + """Create a complete power-of-two DXT mip chain with deterministic byte payloads.""" + mips: list[TPCMipmap] = [] + level = 0 + while True: + mip_width = max(1, width >> level) + mip_height = max(1, height >> level) + mip_size = tpc_format.get_size(mip_width, mip_height) + mips.append( + TPCMipmap( + mip_width, + mip_height, + tpc_format, + bytearray([((seed + level) % 251) + 1]) * mip_size, + ) + ) + if mip_width == 1 and mip_height == 1: + return mips + level += 1 + + +def _animated_dxt1_tpc() -> TPC: + """Build a 2x2 cycle texture shaped like PLC_FrcDist02.tpc.""" + tpc = TPC() + tpc._format = TPCTextureFormat.DXT1 # noqa: SLF001 + tpc.layers = [ + TPCLayer(_dxt_mip_chain(128, 128, TPCTextureFormat.DXT1, seed)) + for seed in range(4) + ] + tpc.is_animated = True + tpc.txi = ( + "proceduretype cycle\r\n" + "numx 2\r\n" + "numy 2\r\n" + "fps 16\r\n" + "blending additive\r\n" + "downsamplemax 0\r\n" + "downsamplemin 0" + ) + return tpc + + +def _total_texture_payload_size(tpc: TPC) -> int: + return sum(len(mipmap.data) for layer in tpc.layers for mipmap in layer.mipmaps) + + class TestTPCData(unittest.TestCase): def setUp(self): self.tpc = TPC() @@ -198,6 +244,58 @@ def test_tpc_convert_dxt1_to_rgba_preserves_transparency(self): self.assertEqual(mipmap.tpc_format, TPCTextureFormat.RGBA) self.assertTrue(all(alpha == 0 for alpha in mipmap.data[3::4])) + def test_animated_tpc_writer_uses_retail_txi_footer_layout(self): + """Cycle TPCs must put TXI immediately after the packed frame/mip payload.""" + tpc = _animated_dxt1_tpc() + total_payload_size = _total_texture_payload_size(tpc) + + raw = bytes_tpc(tpc, ResourceType.TPC) + + self.assertEqual(total_payload_size, struct.unpack_from(" None: self.assertEqual(utc.computer_use, 0) self.assertEqual(utc.treat_injury, 0) + def test_dismantle_utc_fills_removed_feat_slot_with_added_feat(self) -> None: + """Replacing one feat should not shift every later FeatList entry.""" + source_xml = """ + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + 10 + 28 + 55 + + + + + + """ + gff = read_gff(source_xml.encode("utf-8"), file_format=ResourceType.GFF_XML) + utc = construct_utc(gff) + + # This mirrors the editor: checked values come back in widget/display order, not in + # the original GFF list order. Feat 28 was removed and feat 101 was added. + utc.feats = [10, 55, 101] + + rebuilt = dismantle_utc(utc) + feat_list = rebuilt.root.get_list("FeatList") + self.assertIsNotNone(feat_list) + assert feat_list is not None + self.assertEqual([entry.get_uint16("Feat") for entry in feat_list], [10, 101, 55]) + + def test_dismantle_utc_does_not_materialize_absent_optional_defaults(self) -> None: + """Default-valued optional fields absent in the source should stay absent on save.""" + source_xml = """ + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + + + + + """ + gff = read_gff(source_xml.encode("utf-8"), file_format=ResourceType.GFF_XML) + rebuilt = dismantle_utc(construct_utc(gff)) + + for label in ( + "Portrait", + "SaveWill", + "SaveFortitude", + "Morale", + "MoraleRecovery", + "MoraleBreakpoint", + ): + self.assertFalse(rebuilt.root.exists(label), label) + def test_read_utc_validates_binary_utc_header(self) -> None: """read_utc runs Kaitai GFF parse and requires header type ``UTC `` for binary data.""" gff = read_gff(TEST_UTC_XML.encode("utf-8"), file_format=ResourceType.GFF_XML) From ba5b4f26d3351214c049d5373e617d0cd3f9e4fe Mon Sep 17 00:00:00 2001 From: Vriff <74007411+vrifftech@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:45:25 -0400 Subject: [PATCH 2/2] Fixed the fix --- .../pykotor/resource/formats/tpc/io_tpc.py | 46 +++---------------- .../pykotor/resource/formats/txi/txi_data.py | 20 ++++++++ .../tests/resource/formats/test_tpc.py | 19 ++++++++ .../tests/resource/formats/test_txi_data.py | 5 ++ 4 files changed, 50 insertions(+), 40 deletions(-) diff --git a/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py b/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py index 20c9f7df5..decde8e32 100644 --- a/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py +++ b/Libraries/PyKotor/src/pykotor/resource/formats/tpc/io_tpc.py @@ -108,37 +108,6 @@ class TPCBinaryReader(ResourceReader): MAX_DIMENSIONS: Literal[0x8000] = 0x8000 IMG_DATA_START_OFFSET: Literal[0x80] = 0x80 - TXI_COMMAND_HINTS = frozenset( - { - "alphamean", - "blending", - "bumpmapscaling", - "bumpmaptexture", - "bumpyshinytexture", - "clamp", - "compresstexture", - "cube", - "decal", - "defaultheight", - "defaultwidth", - "distort", - "distortionamplitude", - "downsamplemax", - "downsamplemin", - "envmaptexture", - "fps", - "isbumpmap", - "islightmap", - "mipmap", - "numx", - "numy", - "proceduretype", - "speed", - "waterheight", - "waterwidth", - "xbox_downsample", - } - ) def __init__( self, @@ -409,17 +378,14 @@ def _looks_like_txi_at(cls, data: bytes, offset: int) -> bool: if any(byte < 32 and byte not in b"\r\n\t" for byte in sample): return False - text = raw.decode("ascii", errors="ignore").strip() - if not text: + try: + text = raw.decode("ascii").strip() + except UnicodeDecodeError: return False - for line in text.splitlines(): - stripped = line.strip() - if not stripped: - continue - command = stripped.split(None, maxsplit=1)[0].lower() - return command in cls.TXI_COMMAND_HINTS - return False + from pykotor.resource.formats.txi.txi_data import TXI + + return TXI.looks_like_txi(text) def _normalize_cubemaps(self): self._tpc.convert( diff --git a/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py b/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py index 35950a570..44f0dca45 100644 --- a/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py +++ b/Libraries/PyKotor/src/pykotor/resource/formats/txi/txi_data.py @@ -331,6 +331,26 @@ def empty(self) -> bool: def get_features(self) -> TXIFeatures: return self.features + @staticmethod + def looks_like_txi(text: str) -> bool: + """Return whether text begins with a recognized TXI command. + + TPC readers use this for embedded-footer detection after they have + already selected a structurally plausible footer offset. Keep command + knowledge centralized here instead of duplicating TXI keyword hints in + each container parser. + """ + for line in text.replace("\x00", "").splitlines(): + parsed_line = line.strip() + if not parsed_line: + continue + + raw_cmd = parsed_line.split(None, maxsplit=1)[0].strip().upper() + if raw_cmd == "DECAL1": # per_lt06.tpc, per_lt07.tpc + raw_cmd = "DECAL" + return raw_cmd in TXICommand.__members__ + return False + @staticmethod def parse_blending(s: str) -> int: s_norm = (s or "").strip().lower() diff --git a/Libraries/PyKotor/tests/resource/formats/test_tpc.py b/Libraries/PyKotor/tests/resource/formats/test_tpc.py index c3da39101..0ac1030f2 100644 --- a/Libraries/PyKotor/tests/resource/formats/test_tpc.py +++ b/Libraries/PyKotor/tests/resource/formats/test_tpc.py @@ -296,6 +296,25 @@ def test_tpc_reader_recovers_txi_from_legacy_animated_header(self): self.assertIn("blending additive", parsed.txi) self.assertNotIn("blending 1", parsed.txi) + def test_tpc_reader_detects_full_txi_command_set_at_declared_footer(self): + """Footer detection must not depend on a TPC-local curated TXI hint list.""" + payload = b"\x00" * TPCTextureFormat.DXT1.get_size(4, 4) + txi = b"maxSizeHQ 64" + raw = bytearray() + raw.extend(struct.pack("