#!/usr/bin/env python3 """ Compact Translation Extractor for Character-Limited AI Translation Outputs untranslated entries in minimal JSON format with whitespace stripped. TOML format only. """ import json import sys from pathlib import Path import argparse import tomllib # Python 3.11+ (stdlib) class CompactTranslationExtractor: def __init__( self, locales_dir: str = "frontend/public/locales", ignore_file: str = "scripts/ignore_translation.toml", ): self.locales_dir = Path(locales_dir) self.golden_truth_file = self.locales_dir / "en-GB" / "translation.toml" if not self.golden_truth_file.exists(): print( f"Error: en-GB translation file not found at {self.golden_truth_file}", file=sys.stderr, ) sys.exit(1) self.golden_truth = self._load_translation_file(self.golden_truth_file) self.ignore_file = Path(ignore_file) self.ignore_patterns = self._load_ignore_patterns() def _load_translation_file(self, file_path: Path) -> dict: """Load TOML translation file.""" try: with open(file_path, "rb") as f: return tomllib.load(f) except FileNotFoundError: print(f"Error: File not found: {file_path}", file=sys.stderr) sys.exit(1) except Exception as e: print(f"Error: Invalid TOML file {file_path}: {e}", file=sys.stderr) sys.exit(1) def _load_ignore_patterns(self) -> dict: """Load ignore patterns from TOML file.""" if not self.ignore_file.exists(): return {} try: with open(self.ignore_file, "rb") as f: ignore_data = tomllib.load(f) return { lang: set(data.get("ignore", [])) for lang, data in ignore_data.items() } except Exception as e: print( f"Warning: Could not load ignore file {self.ignore_file}: {e}", file=sys.stderr, ) return {} def _flatten_dict( self, d: dict, parent_key: str = "", separator: str = "." ) -> dict: """Flatten nested dictionary into dot-notation keys.""" items = [] for k, v in d.items(): new_key = f"{parent_key}{separator}{k}" if parent_key else k if isinstance(v, dict): items.extend(self._flatten_dict(v, new_key, separator).items()) else: items.append((new_key, str(v))) return dict(items) def get_untranslated_entries(self, language: str) -> dict: """Get all untranslated entries for a language in compact format.""" lang_dir = self.locales_dir / language target_file = lang_dir / "translation.toml" if not target_file.exists(): print( f"Error: Translation file not found for language: {language}", file=sys.stderr, ) sys.exit(1) target_data = self._load_translation_file(target_file) golden_flat = self._flatten_dict(self.golden_truth) target_flat = self._flatten_dict(target_data) lang_code = language.replace("-", "_") ignore_set = self.ignore_patterns.get(lang_code, set()) # Find missing translations missing_keys = set(golden_flat.keys()) - set(target_flat.keys()) - ignore_set # Find untranslated entries (identical to en-GB or marked [UNTRANSLATED]) untranslated_keys = set() for key in target_flat: if key in golden_flat and key not in ignore_set: target_value = target_flat[key] golden_value = golden_flat[key] if ( isinstance(target_value, str) and target_value.startswith("[UNTRANSLATED]") ) or ( golden_value == target_value and not self._is_expected_identical(key, golden_value) ): untranslated_keys.add(key) # Combine and create compact output all_untranslated = missing_keys | untranslated_keys compact_entries = {} for key in sorted(all_untranslated): if key in golden_flat: compact_entries[key] = golden_flat[key] return compact_entries def _is_expected_identical(self, key: str, value: str) -> bool: """Check if a key-value pair is expected to be identical across languages.""" identical_patterns = ["language.direction"] identical_values = {"ltr", "rtl", "True", "False", "true", "false", "unknown"} if value.strip() in identical_values: return True for pattern in identical_patterns: if pattern in key.lower(): return True return False def main(): parser = argparse.ArgumentParser( description="Extract untranslated entries in compact format for AI translation (TOML format only)" ) parser.add_argument("language", help="Language code (e.g., de-DE, fr-FR)") parser.add_argument( "--locales-dir", default="frontend/public/locales", help="Path to locales directory", ) parser.add_argument( "--ignore-file", default="scripts/ignore_translation.toml", help="Path to ignore patterns file", ) parser.add_argument( "--max-entries", type=int, help="Maximum number of entries to output" ) parser.add_argument("--output", help="Output file (default: stdout)") args = parser.parse_args() extractor = CompactTranslationExtractor(args.locales_dir, args.ignore_file) untranslated = extractor.get_untranslated_entries(args.language) if args.max_entries: # Take first N entries keys = list(untranslated.keys())[: args.max_entries] untranslated = {k: untranslated[k] for k in keys} # Output compact JSON (no indentation, minimal whitespace) output = json.dumps(untranslated, separators=(",", ":"), ensure_ascii=False) if args.output: with open(args.output, "w", encoding="utf-8") as f: f.write(output) print( f"Extracted {len(untranslated)} untranslated entries to {args.output}", file=sys.stderr, ) else: print(output) if __name__ == "__main__": main()