diff --git a/README.md b/README.md index 68c7d2668..0e1fe9bad 100644 --- a/README.md +++ b/README.md @@ -127,3 +127,5 @@ Markdown files in the content directly can be [edited directly on GitHub](https: (synchronize the translated file structure with the English `index.md` source). Or create an issue/ [send us](mailto:hello@organicmaps.app) the text, and we'll add translations. New language should be also added in `config.toml`. + +Run `python3 tools/translate_check.py source.md translated.md LANG` before publishing a translation. The checker reports malformed YAML, missing or empty reader-visible metadata, and changed product names or runtime placeholders. Frontmatter and its `extra` field must be mappings. The FAQ section's maintenance note is exempt from body checks only with `render: false` in both the source and translation; its visible metadata is always checked. Translation and Telegram commands reject malformed frontmatter before making API calls. diff --git a/content/news/2024-03-22/325/index.ru.md b/content/news/2024-03-22/325/index.ru.md index 016431630..318b047c8 100644 --- a/content/news/2024-03-22/325/index.ru.md +++ b/content/news/2024-03-22/325/index.ru.md @@ -1,5 +1,5 @@ --- -Название: "Давайте вместе создадим лучшие карты для мобильных устройств!" +title: "Давайте вместе создадим лучшие карты для мобильных устройств!" date: 2024-03-22T14:51:39+00:00 slug: "lets-build-the-best-maps-app-together" --- diff --git a/tmp/test_tooling.py b/tmp/test_tooling.py index 5e8a37d53..b2fe29766 100644 --- a/tmp/test_tooling.py +++ b/tmp/test_tooling.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 """Focused, offline regressions for translation, hooks and Telegram tooling.""" +import io import json import subprocess import sys import tempfile import unittest +from contextlib import redirect_stderr from pathlib import Path from unittest.mock import patch from xml.etree import ElementTree @@ -14,6 +16,7 @@ sys.path.insert(0, str(ROOT / "tools")) from deepl_glossary import dictionary_for_probe, load_terms # noqa: E402 +from markdown_frontmatter import FrontmatterError, strip_frontmatter # noqa: E402 from telegram_post import ( # noqa: E402 classify_media, convert_markdown_to_telegramv2, @@ -24,7 +27,9 @@ validate_media_set, visible_text, ) -from translate_check import ERROR, check_translation, emphasis_faults # noqa: E402 +from translate_check import ( # noqa: E402 + ERROR, check_translation, emphasis_faults, syntax_faults, +) from translate_md import ( # noqa: E402 _segments, localize_zola_links, register_ok, tidy, translate_post, translate_text, ) @@ -39,6 +44,140 @@ def json(self) -> dict: class TranslationToolingTests(unittest.TestCase): + def test_malformed_frontmatter_is_rejected_in_both_documents(self): + source = "---\ntitle: Help\n---\n\nOrganic Maps.\n" + for header in ("title: [unclosed", "date: 2026-13-01", "- Help", "[]", "false", "0", + "extra: [Help]", "extra: false", "extra: null"): + with self.subTest(header=header): + broken = f"---\n{header}\n---\n\nOrganic Maps.\n" + with self.assertRaises(FrontmatterError): + strip_frontmatter(broken) + with self.assertRaises(FrontmatterError): + strip_frontmatter(f"---\n{header}\n---") + self.assertTrue(syntax_faults(Path("index.md"), broken, set())) + for src, out in ((source, broken), (broken, source)): + problems = check_translation(src, out, "de") + self.assertEqual([(p.level, p.code) for p in problems], + [(ERROR, "frontmatter")]) + self.assertEqual(strip_frontmatter("Organic Maps.\n"), + ("Organic Maps.\n", {})) + self.assertEqual(strip_frontmatter("---\n---\nOrganic Maps.\n"), + ("Organic Maps.\n", {})) + self.assertEqual(strip_frontmatter("---\r\ntitle: Help\r\n---"), + ("", {"title": "Help"})) + self.assertEqual(check_translation("Organic Maps.\n", "Organic Maps.\n", "de"), []) + + def test_source_frontmatter_strings_must_remain_present(self): + source = """--- +title: Help +extra: + form: + submit_once: Donate {amount} +--- + +Organic Maps. +""" + for header in ("", "---\n\n---\n\n", "---\nrender: false\n---\n\n"): + with self.subTest(header=header): + problems = check_translation(source, header + "Organic Maps.\n", "de") + self.assertEqual( + {(p.level, p.code, p.message) for p in problems}, + {(ERROR, "frontmatter-field", f"'{key}' is missing or is not a string") + for key in ("title", "extra.form.submit_once")}, + ) + for title in ('title: ""', 'title: " "', "title: 42", "Название: Hilfe"): + with self.subTest(title=title): + translated = source.replace("title: Help", title).replace( + "Donate {amount}", "Spende {amount}") + problems = check_translation(source, translated, "de") + self.assertEqual([(p.level, p.code) for p in problems], + [(ERROR, "frontmatter-field")]) + + def test_brands_are_checked_in_metadata_and_keep_their_field_context(self): + source = """--- +title: Organic Maps +description: Organic Maps +extra: + preview_image: Organic Maps.png + translation_diverges: Organic Maps + form: + help: Organic Maps +--- + +Organic Maps. +""" + for original, replacement, field in ( + ("title: Organic Maps", "title: Organische Karten", "title"), + ("description: Organic Maps", "description: Karten", "description"), + ("help: Organic Maps", "help: Karten", "extra.form.help"), + ): + with self.subTest(field=field): + problems = check_translation(source, source.replace(original, replacement), "de") + errors = [p for p in problems if p.level == ERROR] + self.assertEqual([p.code for p in errors], ["brand-lost"]) + self.assertIn(f"'{field}'", errors[0].message) + translated = source.replace("Organic Maps.png", "Karten.png").replace( + "translation_diverges: Organic Maps", "translation_diverges: Andere Fassung") + self.assertFalse(any(p.level == ERROR for p in check_translation(source, translated, "de"))) + quoted = source.replace("title: Organic Maps", "title: '«Organic Maps»'") + self.assertIn("brand-quoted", {p.code for p in check_translation(source, quoted, "de")}) + translated_brand = source.replace( + "description: Organic Maps", "description: Organic Maps e mappe organiche") + self.assertIn("brand-translated", { + p.code for p in check_translation(source, translated_brand, "it")}) + + def test_only_unrendered_taxonomy_notes_skip_body_checks(self): + source = """--- +render: false +title: Help +--- + +### This page is replaced with taxonomy "faq" from templates/faq/list.html +""" + translated = source.replace("title: Help", "title: Hilfe") + self.assertEqual(check_translation(source, translated, "de"), []) + for source_render, target_render in ( + ("render: true", "render: true"), + ("render: 'false'", "render: 'false'"), + ("", ""), + ("render: false", "render: true"), + ("render: false", ""), + ): + with self.subTest(source_render=source_render, target_render=target_render): + rendered = source.replace("render: false", source_render) + missing = translated.replace("render: false", target_render).split("###")[0] + self.assertIn("structure", {p.code for p in check_translation(rendered, missing, "de")}) + missing_metadata = "---\nrender: false\n---\n\n" + translated.split("---\n", 2)[2] + self.assertIn("frontmatter-field", { + p.code for p in check_translation(source, missing_metadata, "de")}) + + def test_translation_cli_rejects_invalid_source_before_api_calls(self): + import translate_md + with tempfile.TemporaryDirectory() as folder: + source = Path(folder) / "index.md" + source.write_text("---\ntitle: [unclosed\n---\n\nBody.\n", encoding="utf-8") + errors = io.StringIO() + with patch.object(sys, "argv", ["translate_md.py", str(source), "--langs", "de"]), \ + patch("translate_md.get_glossary_id", return_value=None) as glossary, \ + patch("translate_md.deepl_translate") as request, redirect_stderr(errors): + with self.assertRaises(SystemExit) as exit: + translate_md.main() + self.assertEqual(exit.exception.code, 2) + self.assertIn(str(source), errors.getvalue()) + glossary.assert_not_called() + request.assert_not_called() + self.assertFalse(source.with_name("index.de.md").exists()) + + def test_proofreading_rejects_invalid_source_before_calling_the_model(self): + from translate_review import review_translation + source = "---\n- invalid metadata\n---\n\nOrganic Maps.\n" + draft = "Organic Maps.\n" + with patch("translate_review.subprocess.run") as request: + fixed, note = review_translation(source, draft, "de") + self.assertEqual(fixed, draft) + self.assertIn("invalid source frontmatter", note) + request.assert_not_called() + def test_translation_imports_do_not_require_requests(self): code = """ import importlib.abc, sys @@ -176,12 +315,13 @@ def test_frontmatter_cleanup_preserves_code_and_extra_fields(self): Body. ''' - with patch("translate_md.brands_visible", return_value=False), \ - patch("translate_md.get_glossary_id", return_value=None), \ - patch("translate_md.deepl_translate", side_effect=lambda texts, *a, **k: (texts, None)): - translated = translate_post(source, "fr", use_glossary=False) - from markdown_frontmatter import strip_frontmatter - self.assertEqual(strip_frontmatter(source)[1], strip_frontmatter(translated)[1]) + for document in (source, source.split("---\n\nBody.")[0] + "---"): + with self.subTest(document=document): + with patch("translate_md.brands_visible", return_value=False), \ + patch("translate_md.get_glossary_id", return_value=None), \ + patch("translate_md.deepl_translate", side_effect=lambda texts, *a, **k: (texts, None)): + translated = translate_post(document, "fr", use_glossary=False) + self.assertEqual(strip_frontmatter(document)[1], strip_frontmatter(translated)[1]) def test_links_localize_without_translating_protected_examples(self): target = "@/contribute/index.md" @@ -274,6 +414,37 @@ def test_probe_builds_unknown_and_variant_targets(self): class TelegramToolingTests(unittest.TestCase): + def test_invalid_frontmatter_aborts_all_posts_before_api_calls(self): + import telegram_post + import telegram_post_all + with tempfile.TemporaryDirectory() as folder: + folder = Path(folder) + (folder / "index.md").write_text("Organic Maps.\n", encoding="utf-8") + broken = folder / "index.de.md" + bad_header = "---\nextra: [invalid]\n---\n\nBody.\n" + target = folder / "content/target/index.md" + target.parent.mkdir(parents=True) + target.write_text(bad_header, encoding="utf-8") + for content, invalid_path in ( + (bad_header, broken), + ("Read [the page](@/target/index.md).\n", target), + ): + broken.write_text(content, encoding="utf-8") + for module, args in ( + (telegram_post, ["--text", str(broken), "--group", "@example", "--yes"]), + (telegram_post_all, [str(folder), "--only", "en,de", "--yes"]), + ): + with self.subTest(module=module.__name__, invalid_path=invalid_path): + errors = io.StringIO() + with patch.object(sys, "argv", [module.__name__, *args]), \ + patch.object(module, "__file__", str(folder / "tools/post.py")), \ + patch("telegram_post.requests.post") as request, redirect_stderr(errors): + with self.assertRaises(SystemExit) as exit: + module.main() + self.assertEqual(exit.exception.code, 2) + self.assertIn(str(invalid_path), errors.getvalue()) + request.assert_not_called() + def test_current_tera_components_are_removed_from_posts(self): text = ( "Before\n\n{{ tuple[str, dict]: """Return ``(body, metadata)`` for a Markdown document. - Invalid YAML yields empty metadata while still returning the body, matching - the historical behaviour of the publishing and translation tools. + Missing or null frontmatter produces empty metadata. Invalid YAML or a + non-mapping metadata/extra value raises FrontmatterError so a broken + header cannot silently disappear from translation or publishing. """ - match = re.match(r"^---\s*\n(.*?\n)---\s*\n", text, re.DOTALL) + match = FRONTMATTER_RE.match(text) if not match: return text, {} try: - metadata = yaml.safe_load(match.group(1)) or {} - except yaml.YAMLError: + metadata = yaml.safe_load(match.group(1)) + except (yaml.YAMLError, ValueError) as error: + raise FrontmatterError(f"invalid YAML frontmatter: {error}") from error + if metadata is None: metadata = {} + if not isinstance(metadata, dict): + raise FrontmatterError("frontmatter must be a mapping") + if "extra" in metadata and not isinstance(metadata["extra"], dict): + raise FrontmatterError("frontmatter 'extra' must be a mapping") return text[match.end() :], metadata diff --git a/tools/telegram_post.py b/tools/telegram_post.py index eff328350..7320fe16f 100755 --- a/tools/telegram_post.py +++ b/tools/telegram_post.py @@ -27,7 +27,7 @@ import requests -from markdown_frontmatter import strip_frontmatter +from markdown_frontmatter import FrontmatterError, strip_frontmatter from markdown_xml import find_balanced_close BASE_URL = "https://api.telegram.org/bot{token}" @@ -207,7 +207,8 @@ def _zola_target_url( Reads the target file's frontmatter to honor a `slug` override and prepends the language code as a URL prefix for non-default languages. - Returns None if the file can't be found. + Returns None if the file can't be found. Malformed target metadata raises + FrontmatterError because its slug cannot be resolved reliably. """ target = target.lstrip("/") file_path = site_root / "content" / target @@ -234,6 +235,8 @@ def _zola_target_url( slug = raw_slug.strip() except OSError: pass + except FrontmatterError as error: + raise FrontmatterError(f"{file_path}: {error}") from error # Determine the trailing path segment: # - If the file is an _index.* (section) or index.* (page), the URL is the directory. @@ -1120,12 +1123,13 @@ def main(): print(f"Error: {media_problem}", file=sys.stderr) sys.exit(1) - token = get_token() - chat_id = resolve_chat_id(token, args.group) raw_text = args.text.read_text(encoding="utf-8") # Strip frontmatter; use "title" field as a bold header - text, meta = strip_frontmatter(raw_text) + try: + text, meta = strip_frontmatter(raw_text) + except FrontmatterError as error: + parser.error(f"{args.text}: {error}") title = meta.get("title", "") if title: text = f"**{title}**\n\n{text.lstrip()}" @@ -1149,7 +1153,10 @@ def main(): # Resolve Zola @/-style internal references → absolute site URLs. base_url = load_base_url(site_root) - text, zola_unresolved = resolve_zola_references(text, site_root, base_url) + try: + text, zola_unresolved = resolve_zola_references(text, site_root, base_url) + except FrontmatterError as error: + parser.error(str(error)) if zola_unresolved: print( f"\nWarning: {len(zola_unresolved)} Zola @/-reference(s) could not be resolved:", @@ -1180,6 +1187,8 @@ def main(): print("Aborted.") sys.exit(0) + token = get_token() + chat_id = resolve_chat_id(token, args.group) print(f"Target chat: {chat_id}") if title: print(f"Title: {title}") diff --git a/tools/telegram_post_all.py b/tools/telegram_post_all.py index e2a98c585..57b56bc72 100755 --- a/tools/telegram_post_all.py +++ b/tools/telegram_post_all.py @@ -23,7 +23,7 @@ import sys from pathlib import Path -from markdown_frontmatter import strip_frontmatter +from markdown_frontmatter import FrontmatterError, strip_frontmatter from telegram_post import ( get_token, resolve_chat_id, @@ -128,7 +128,10 @@ def prepare_text(md_path: Path, site_root: Path) -> str: """Read markdown file, strip frontmatter, resolve references, clean up.""" raw = md_path.read_text(encoding="utf-8") - text, meta = strip_frontmatter(raw) + try: + text, meta = strip_frontmatter(raw) + except FrontmatterError as error: + raise FrontmatterError(f"{md_path}: {error}") from error title = meta.get("title", "") if title: text = f"**{title}**\n\n{text.lstrip()}" @@ -214,16 +217,17 @@ def main() -> None: print(f"Error: {media_problem}", file=sys.stderr) sys.exit(1) - # A dry run never talks to the API, so it must not require a token. - token = "" if args.dry_run else get_token() - # Discover which groups have matching markdown files, and render each post # once — prepare_text() is not free and its warnings should print once. tasks: list[tuple[str, Path, str]] = [] for group, filename in groups.items(): md_path = args.folder / filename if md_path.is_file(): - tasks.append((group, md_path, prepare_text(md_path, site_root))) + try: + text = prepare_text(md_path, site_root) + except FrontmatterError as error: + parser.error(str(error)) + tasks.append((group, md_path, text)) if not tasks: print("No matching markdown files found in the folder.", file=sys.stderr) @@ -276,9 +280,10 @@ def main() -> None: print("Aborted.") sys.exit(0) - # Resolve every chat up front. resolve_chat_id() exits on an unknown - # channel, and doing that mid-run would leave a half-published post with - # nothing posted yet, this is a safe abort. + # Prepare all files and resolve all chats before sending anything, so a + # bad header or unknown channel cannot leave a partly published post. + # A dry run needs neither credentials nor API calls. + token = "" if args.dry_run else get_token() chat_ids: dict[str, str] = {} if not args.dry_run: print("Resolving channels...") diff --git a/tools/translate_check.py b/tools/translate_check.py index e36eed395..d31d20a62 100755 --- a/tools/translate_check.py +++ b/tools/translate_check.py @@ -27,7 +27,7 @@ from dataclasses import dataclass from pathlib import Path -from markdown_frontmatter import strip_frontmatter +from markdown_frontmatter import FrontmatterError, strip_frontmatter from translate_md import QUOTE_FOR, register_ok ERROR, WARN = "ERROR", "warn" @@ -80,7 +80,7 @@ def fingerprint(text: str) -> dict[str, int]: BRANDS = ["Organic Maps", "OpenStreetMap", "ID Editor", "Google Play", "App Store", "F-Droid", "Obtainium", "Accrescent", "TestFlight", - "Huawei AppGallery", "AppGallery", "Weblate", "TestFlight"] + "Huawei AppGallery", "AppGallery", "Weblate"] QUOTE_CHARS = "\"'«»„“”‘’《》〈〉「」〔〕" # Opening/closing marks that genuinely pair, so a stray apostrophe used as # a suffix separator is not mistaken for a closing quote. @@ -88,17 +88,6 @@ def fingerprint(text: str) -> dict[str, int]: ("“", "”"), ("‘", "’"), ("《", "》"), ("〈", "〉"), ("「", "」"), ("〔", "〕")] -# Which brands a language is allowed to transliterate. Measured, not assumed: -# he/ar/fa-IR keep Organic Maps, OpenStreetMap and iOS in Latin but render -# Android natively; the Indic scripts transliterate everything. -BRAND_TRANSLITERATING = { - "he": {"Android"}, "ar": {"Android"}, "fa-IR": {"Android"}, - # hi, ml and te used to be exempt from every brand check because they - # transliterated the product name. That was decided against: they now keep - # Latin brands like everyone else, so only Android stays exempt. - "hi": {"Android"}, "ml": {"Android"}, "te": {"Android"}, -} - # The product name rendered as translated words rather than kept in Latin. # Every pattern here was observed in the corpus, not guessed: a review pass # found 165 pages across 20 languages doing this. Stems, because most of these @@ -332,7 +321,10 @@ def syntax_faults(path: Path, text: str, shared_refs: set[str]) -> list[str]: translation comparison. Every one of these was a real defect in this corpus, and each rendered as visibly broken output. """ - body, _ = strip_frontmatter(text) + try: + body, _ = strip_frontmatter(text) + except FrontmatterError as error: + return [f"frontmatter: {error}"] out = [f"emphasis: {f}" for f in emphasis_faults(body)] for match in re.finditer(r"\]\s+\((?:https?://|@/|/|mailto:|#)", body): @@ -507,17 +499,95 @@ def _declared_source_keeps( return keeps, faults -def check_translation(src: str, out: str, lang: str) -> list[Problem]: - """Compare a translated markdown body against its English source.""" - src_body, _ = strip_frontmatter(src) - out_body, out_meta = strip_frontmatter(out) +def _brand_problems(src: str, out: str, lang: str) -> list[Problem]: + """Product names stay Latin and unquoted in body text and metadata.""" problems: list[Problem] = [] + pat = TRANSLATED_BRAND.get(lang) + if pat and not (lang in TRANSLATED_BRAND_EXEMPT + and TRANSLATED_BRAND_EXEMPT[lang][0] in src): + found = re.findall(pat, out, re.IGNORECASE) + if found: + problems.append(Problem( + ERROR, "brand-translated", + f"the product name appears translated {len(found)}x, not as " + f"'Organic Maps'", + str(sorted({f if isinstance(f, str) else " ".join(f) + for f in found})[:3]))) - # A section _index body is a note to whoever maintains the templates — - # "This page is replaced with taxonomy \"faq\" from templates/faq/list.html". - # No template renders it, so its typography is nobody's concern, and the - # strings it quotes are identifiers that must stay as they are. - if re.search(r"templates/\S+\.html", src_body) and "replaced with taxonomy" in src_body: + for brand in BRANDS: + n_src, n_out = src.count(brand), out.count(brand) + if n_src > n_out: + problems.append(Problem( + ERROR, "brand-lost", + f"'{brand}' appears {n_src}x in source but {n_out}x here")) + # Paired marks quote a brand; a Turkish case apostrophe does not. + if n_out and any(re.search(f"{re.escape(o)}{re.escape(brand)}{re.escape(c)}", out) + for o, c in QUOTE_PAIRS): + problems.append(Problem( + WARN, "brand-quoted", f"'{brand}' is wrapped in quotation marks")) + return problems + + +def _frontmatter_problems(src_meta: dict, out_meta: dict, lang: str) -> list[Problem]: + """Validate visible strings independently of whether the body is rendered.""" + problems: list[Problem] = [] + src_fields = _frontmatter_prose(src_meta) + out_fields = _frontmatter_prose(out_meta) + for key, value in out_fields.items(): + if not value.strip(): + continue + ok, msg = register_ok(value, lang) + if not ok: + problems.append(Problem(WARN, "register-frontmatter", f"'{key}': {msg}")) + for problem in _brand_problems(src_fields.get(key, ""), value, lang): + problem.message = f"'{key}': {problem.message}" + problems.append(problem) + + declared, faults = _declared_source_keeps(out_meta, src_fields, out_fields) + problems.extend(faults) + for key, value in src_fields.items(): + if key not in out_fields: + problems.append(Problem( + ERROR, "frontmatter-field", f"'{key}' is missing or is not a string")) + continue + translated = out_fields[key] + if value.strip() and not translated.strip(): + problems.append(Problem( + ERROR, "frontmatter-field", f"'{key}' must not be empty")) + continue + if (key not in declared and translated == value + and any(c.isalpha() for c in value)): + problems.append(Problem( + WARN, "untranslated-frontmatter", + f"'{key}' is identical to the English source")) + # Form placeholders are literal keys substituted at runtime. + want = sorted(PLACEHOLDER_RE.findall(value)) + got = sorted(PLACEHOLDER_RE.findall(translated)) + if want != got: + problems.append(Problem( + ERROR, "placeholder-mismatch", + f"'{key}' must keep the placeholder(s) verbatim", + f"source has {want or 'none'}, translation has {got or 'none'}")) + return problems + + +def check_translation(src: str, out: str, lang: str) -> list[Problem]: + """Compare translated Markdown and reader-visible metadata against English.""" + parsed = [] + for label, text in (("source", src), ("translation", out)): + try: + parsed.append(strip_frontmatter(text)) + except FrontmatterError as error: + return [Problem(ERROR, "frontmatter", f"{label}: {error}")] + (src_body, src_meta), (out_body, out_meta) = parsed + problems = _frontmatter_problems(src_meta, out_meta, lang) + + # An unrendered FAQ section documents the taxonomy that owns its URL. + # Rendered content still needs body checks even when it quotes that note. + if (src_meta.get("render") is False + and out_meta.get("render") is False + and re.search(r"templates/\S+\.html", src_body) + and "replaced with taxonomy" in src_body): return problems # 1. Structure must match the source exactly — unless the page says it @@ -680,42 +750,7 @@ def kind(text: str) -> Counter: ERROR, "contributor-name-lost", f"contributor '{name}' from the source does not appear here")) - # 5d. The product name translated into local words. Distinct from - # brand-lost: the page may also use the Latin form elsewhere, so a - # count check cannot see it. - pat = TRANSLATED_BRAND.get(lang) - if pat and not (lang in TRANSLATED_BRAND_EXEMPT - and TRANSLATED_BRAND_EXEMPT[lang][0] in src_body): - found = re.findall(pat, out_body, re.IGNORECASE) - if found: - problems.append(Problem( - ERROR, "brand-translated", - f"the product name appears translated {len(found)}x, not as " - f"'Organic Maps'", - str(sorted({f if isinstance(f, str) else " ".join(f) - for f in found})[:3]))) - - # 6. Brands: same count, Latin, and never wrapped in quotes. - exempt = BRAND_TRANSLITERATING.get(lang, set()) - if exempt is not None: - for brand in BRANDS: - if brand in exempt: - continue - n_src, n_out = src_body.count(brand), out_body.count(brand) - if n_src and n_out < n_src: - problems.append(Problem( - ERROR, "brand-lost", - f"'{brand}' appears {n_src}x in source but {n_out}x here", - )) - # The two marks must actually pair. Turkish writes the shipped label - # "OpenStreetMap'e Yer Ekle", where the apostrophe attaches the case suffix - # and the quotes belong to the whole label, not to the brand. - for brand in BRANDS: - if any(re.search(f"{re.escape(o)}{re.escape(brand)}{re.escape(c)}", out_body) - for o, c in QUOTE_PAIRS): - problems.append(Problem( - WARN, "brand-quoted", - f"'{brand}' is wrapped in quotation marks")) + problems.extend(_brand_problems(src_body, out_body, lang)) # 7. Words spliced from two alphabets. mixed = mixed_script_words(out_body) @@ -730,57 +765,6 @@ def kind(text: str) -> Counter: if not ok: problems.append(Problem(WARN, "register", msg)) - # 9. Untranslated frontmatter fields, `extra` included. - if out_meta: - src_fields = _frontmatter_prose(strip_frontmatter(src)[1] or {}) - out_fields = _frontmatter_prose(out_meta) - - for key, value in out_fields.items(): - ok, msg = register_ok(value, lang) - if not ok: - problems.append(Problem(WARN, "register-frontmatter", f"'{key}': {msg}")) - - # Some words really are the same in both languages — Dutch and Italian - # write "Privacy", German writes "Name (optional)" — and no textual test - # can tell those from a field the translator simply skipped: Italian - # "Donate" and Occitan "News" were exactly that, and looked identical. - # The page declares the fields whose source wording is intentional: - # - # extra: - # translation_keeps_source: ["extra.menu_title"] - # - # Paste the path out of the warning. The declaration is verified, not - # merely obeyed: it must name a field that exists and is still identical, - # so it cannot outlive the situation it describes. - declared, faults = _declared_source_keeps(out_meta, src_fields, out_fields) - problems.extend(faults) - - for key, value in src_fields.items(): - # A value with no letters reads the same in every language: post 39 - # is called "19:36", a timestamp. - if (key not in declared - and out_fields.get(key) == value - and any(c.isalpha() for c in value)): - problems.append(Problem( - WARN, "untranslated-frontmatter", - f"'{key}' is identical to the English source")) - - # 9a. Runtime placeholders in those same fields. `donate-subscribe.js` - # fills the submit label and the range error by literal string - # replace — `pattern.replace('{amount}', …)` — so a placeholder the - # translator inflected, spaced or dropped does not raise anything: - # the button just reads "Doe {amount}" to every donor in that - # language. Parity with the source is the whole contract, so a - # mismatch is an ERROR. - for key, value in src_fields.items(): - want = sorted(PLACEHOLDER_RE.findall(value)) - got = sorted(PLACEHOLDER_RE.findall(out_fields.get(key, ""))) - if want != got: - problems.append(Problem( - ERROR, "placeholder-mismatch", - f"'{key}' must keep the placeholder(s) verbatim", - f"source has {want or 'none'}, translation has {got or 'none'}")) - # 10. Straight ASCII quotes where the language wants its own marks — but # only in prose. HTML attributes, Tera expressions, code spans and # markdown link titles all *require* a straight quote, so warning about diff --git a/tools/translate_md.py b/tools/translate_md.py index e89bba393..a3eb3197b 100755 --- a/tools/translate_md.py +++ b/tools/translate_md.py @@ -40,7 +40,7 @@ apply_repairs, DEEPL_TARGET, ) -from markdown_frontmatter import strip_frontmatter +from markdown_frontmatter import FRONTMATTER_RE, FrontmatterError, strip_frontmatter REPO_ROOT = Path(__file__).resolve().parent.parent # tools/ sits below it BATCH = 45 # DeepL accepts 50 text params per request @@ -414,7 +414,7 @@ def translate_text(body: str, lang: str, use_glossary: bool = True) -> str: def translate_post(raw: str, lang: str, use_glossary: bool = True) -> str: - """Translate a whole markdown file, frontmatter included if present.""" + """Translate Markdown, rejecting malformed frontmatter before API calls.""" body, meta = strip_frontmatter(raw) if not meta: return translate_text(raw, lang, use_glossary) @@ -444,7 +444,7 @@ def translate_post(raw: str, lang: str, use_glossary: bool = True) -> str: # Rewrite the frontmatter line-wise so unrelated fields keep their exact # formatting — round-tripping through a YAML dumper would reflow them. - m = re.match(r"^---\s*\n(.*?\n)---\s*\n", raw, re.DOTALL) + m = FRONTMATTER_RE.match(raw) fm_text = m.group(1) for key, value in new_values.items(): escaped = value.replace("\\", "\\\\").replace('"', '\\"') @@ -494,11 +494,13 @@ def main() -> None: sys.exit(1) out_dir = args.out or args.source.parent - out_dir.mkdir(parents=True, exist_ok=True) raw = args.source.read_text(encoding="utf-8") stem = args.source.name[:-3] if args.source.name.endswith(".md") else args.source.stem - body, meta = strip_frontmatter(raw) + try: + body, meta = strip_frontmatter(raw) + except FrontmatterError as error: + ap.error(f"{args.source}: {error}") payloads, _ = _segments(body if meta else raw, brands_visible(langs[0])) chars = sum(len(p) for p in payloads) print(f"Source: {args.source}") @@ -516,6 +518,7 @@ def main() -> None: print(f" {p[:150]}") return + out_dir.mkdir(parents=True, exist_ok=True) failures, blocked = [], [] for i, lang in enumerate(langs, 1): dest = out_dir / f"{stem}.{lang}.md" diff --git a/tools/translate_review.py b/tools/translate_review.py index 4faa21a90..fe83e5c33 100755 --- a/tools/translate_review.py +++ b/tools/translate_review.py @@ -9,7 +9,7 @@ corrected version. The result is validated before it is accepted: if the review introduces a -structural error the draft did not have, the draft is kept and the failure is +validation error the draft did not have, the draft is kept and the failure is reported. A proofreader that breaks markdown is worse than none. from translate_review import review_translation @@ -29,6 +29,7 @@ import sys from pathlib import Path +from markdown_frontmatter import FrontmatterError, strip_frontmatter from translate_check import check_translation, ERROR from translate_md import expected_register, detect_register from deepl_glossary import load_terms, get_review_terms @@ -131,9 +132,13 @@ def review_translation(src: str, out: str, lang: str, """Proofread one translation. Returns (text, note). The draft is returned unchanged whenever the review cannot be trusted: - the CLI failed, the output was empty, or it introduced a structural error - the draft did not already have. + the source metadata is invalid, the CLI failed, the output was empty, or + it introduced an error the draft did not already have. """ + try: + strip_frontmatter(src) + except FrontmatterError as error: + return out, f"invalid source frontmatter: {error}" prompt = build_prompt(src, out, lang) try: proc = subprocess.run(