You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2024/01/19 05:52:34 UTC
(superset) 03/04: getting warmer...
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch translation_script
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 8b9876beefde85aee600ae43571dfb7ff250f49c
Author: Evan Rusackas <ev...@rusackas.com>
AuthorDate: Thu Jan 18 15:25:11 2024 -0700
getting warmer...
---
scripts/complete_translations.py | 68 ++++++++++++++++++++++++++++++++++------
1 file changed, 59 insertions(+), 9 deletions(-)
diff --git a/scripts/complete_translations.py b/scripts/complete_translations.py
index 0ac770a715..94479c1bda 100644
--- a/scripts/complete_translations.py
+++ b/scripts/complete_translations.py
@@ -1,5 +1,10 @@
import glob
import polib
+from langchain_community.callbacks import get_openai_callback
+from langchain_openai import OpenAI
+import json
+
+llm = OpenAI(model_name="gpt-3.5-turbo-instruct", n=2, best_of=2)
def read_po_files(glob_pattern):
translations = {}
@@ -45,6 +50,57 @@ def print_missing_translations_with_available(translations, missing_translations
print(f" '{lang}': {trans}")
print(f"Missing translations count: {missing_translations_count}")
+limit = 10;
+
+def generate_missing_translations(translations, missing_translations):
+ with get_openai_callback() as cb:
+ missing_translations_count = 0
+ for msgid, missing_langs in missing_translations.items():
+ for target_lang in missing_langs:
+ missing_translations_count += 1
+
+ template = "I have the following JSON object of `msgid` and `msgstr` strings pulled from a .po file:\n"
+ template += "{\n"
+ template += f"\t\"msgid\": \"{msgid}\",\n"
+ template += "\t\"translations\": {\n"
+
+ translations_items = list(translations[msgid].items())
+ total_items = len(translations_items)
+
+ for index, (lang, trans) in enumerate(translations_items):
+ if trans: # Check if translation is not empty
+ template += f"\t\t\"{lang}\": \"{trans}\""
+ if index < total_items - 1: # If it's not the last item
+ template += ","
+ template += "\n"
+
+ template += "\t}\n"
+ template += "}\n\n"
+ template += f"I'm missing the translation for the '{target_lang}' language\n"
+ template += "Respond with a JSON object and nothing else. The json object should have the `msgid` key/value pair as specified in my JSON object, and the `msgstr` key/value pair with the translated string (including any relevant whitespace)\n"
+ # template += "Respond with the translated `msgstr` value (including any whitespace) and nothing else. The json object should have the `msgid` key/value pair as specified in my JSON object, and the `msgstr` key/value pair with the translated string (including any relevant whitespace)\n"
+
+ if missing_translations_count < 20:
+ result = llm.invoke(template)
+
+ try:
+ result = result.strip()
+ # Parse the corrected string
+ parsed_data = json.loads(result)
+
+ if "msgid" in parsed_data and "msgstr" in parsed_data and parsed_data["msgid"] and parsed_data["msgstr"]:
+ print('SUCCESS!')
+ populate_missing_translation(translations, parsed_data["msgid"], target_lang, parsed_data["msgstr"])
+ except json.JSONDecodeError as e:
+ print(f"Error parsing JSON: {e}")
+ print("Result:", result)
+
+
+
+ print(f"Missing translations count: {missing_translations_count}")
+ print(cb)
+
+
def populate_missing_translation(translations, msgid, language, string):
if msgid in translations:
translations[msgid][language] = string
@@ -58,15 +114,9 @@ missing_translations = find_missing_translations(translations)
# print('Missing translations:')
# print(missing_translations)
-print('Missing translations with available translations:')
-print_missing_translations_with_available(translations, missing_translations)
-
-# Example usage
-msgid = "year"
-language = "sk"
-string = "rok"
-
-populate_missing_translation(translations, msgid, language, string)
+# print('Missing translations with available translations:')
+# print_missing_translations_with_available(translations, missing_translations)
+generate_missing_translations(translations, missing_translations)
update_po_files(glob_pattern, translations)