You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/07/17 04:25:37 UTC

(superset) branch master updated: fix(i18n): accept script subtags (e.g. sr_Latn) in backfill_po language codes (#42103)

This is an automated email from the ASF dual-hosted git repository.

rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git


The following commit(s) were added to refs/heads/master by this push:
     new f467d36a24c fix(i18n): accept script subtags (e.g. sr_Latn) in backfill_po language codes (#42103)
f467d36a24c is described below

commit f467d36a24c4ac087c6e9bc3c75f917a4b8bacd1
Author: Evan Rusackas <ev...@preset.io>
AuthorDate: Thu Jul 16 21:25:30 2026 -0700

    fix(i18n): accept script subtags (e.g. sr_Latn) in backfill_po language codes (#42103)
    
    Co-authored-by: Amin Ghadersohi <am...@gmail.com>
    Co-authored-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
---
 scripts/translations/backfill_po.py                | 24 +++++++++++++++++-----
 .../scripts/translations/backfill_po_test.py       | 18 ++++++++++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/scripts/translations/backfill_po.py b/scripts/translations/backfill_po.py
index 76f863f8955..ee7579592e2 100644
--- a/scripts/translations/backfill_po.py
+++ b/scripts/translations/backfill_po.py
@@ -138,6 +138,22 @@ def _lang_name(code: str) -> str:
     return LANGUAGE_NAMES.get(code, code)
 
 
+# An ISO 639-1/639-2 code, optionally followed by an ISO 3166 region
+# (``_BR``, two uppercase) or an ISO 15924 script (``_Latn``, titlecase). The
+# script subtag matters for catalogs like ``sr_Latn`` (Serbian in Latin script);
+# without it the code was rejected and the catalog silently skipped.
+_LANG_CODE_RE: re.Pattern[str] = re.compile(r"[a-z]{2,3}(_([A-Z]{2}|[A-Z][a-z]{3}))?")
+
+
+def _is_valid_lang_code(lang: str) -> bool:
+    """Validate a language code before it lands, unsanitized, in a filesystem path.
+
+    Guards against path traversal while allowing the region and script subtags
+    Superset actually ships (e.g. ``pt_BR``, ``sr_Latn``).
+    """
+    return bool(_LANG_CODE_RE.fullmatch(lang))
+
+
 def _plural_key(msgid: str, msgid_plural: str) -> str:
     """Build the translation index key used for pluralized entries."""
     return f"{msgid}\x00{msgid_plural}"
@@ -667,13 +683,11 @@ def backfill(
     mark_fuzzy: bool = True,
 ) -> None:
     """Backfill missing translations in the target language's .po file."""
-    # Defense against path traversal: ``lang`` lands in a filesystem path
-    # without further sanitization, so reject anything that isn't an
-    # ISO 639-1/639-2 code with an optional ISO 3166 region (e.g. ``pt_BR``).
-    if not re.fullmatch(r"[a-z]{2,3}(_[A-Z]{2})?", lang):
+    if not _is_valid_lang_code(lang):
         print(
             f"Invalid language code: {lang!r} "
-            "(expected ISO 639 code, optionally with _<REGION>, e.g. 'fr' or 'pt_BR')",
+            "(expected ISO 639 code, optionally with a _<REGION> or _<Script> "
+            "subtag, e.g. 'fr', 'pt_BR', or 'sr_Latn')",
             file=sys.stderr,
         )
         sys.exit(1)
diff --git a/tests/unit_tests/scripts/translations/backfill_po_test.py b/tests/unit_tests/scripts/translations/backfill_po_test.py
index 253c8335b3a..b8223420407 100644
--- a/tests/unit_tests/scripts/translations/backfill_po_test.py
+++ b/tests/unit_tests/scripts/translations/backfill_po_test.py
@@ -40,6 +40,24 @@ backfill_po = importlib.util.module_from_spec(_spec)
 _spec.loader.exec_module(backfill_po)
 
 
+@pytest.mark.parametrize(
+    "lang",
+    ["fr", "de", "pt_BR", "zh_TW", "sr_Latn", "eng"],
+)
+def test_is_valid_lang_code_accepts_region_and_script_subtags(lang: str) -> None:
+    """Region (``pt_BR``) and script (``sr_Latn``) subtags are both valid."""
+    assert backfill_po._is_valid_lang_code(lang)
+
+
+@pytest.mark.parametrize(
+    "lang",
+    ["", "e", "EN", "fr_", "fr_br", "fr_BRA", "../etc", "en_US_x", "sr-Latn"],
+)
+def test_is_valid_lang_code_rejects_malformed_and_traversal(lang: str) -> None:
+    """Malformed codes and path-traversal attempts are rejected."""
+    assert not backfill_po._is_valid_lang_code(lang)
+
+
 def test_parse_response_singular_strings() -> None:
     """A flat object of int-keyed strings is returned as-is."""
     text = '{"0": "hola", "1": "mundo"}'