You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ma...@apache.org on 2021/05/02 15:29:24 UTC

[tika] branch branch_1x updated (7cae30e -> b2f37ef)

This is an automated email from the ASF dual-hosted git repository.

mattmann pushed a change to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git.


    from 7cae30e  TIKA-3374 -- allow users to turn off charset detection of entry names in ZipArchives
     new 2c4ba60  Backport Merge branch 'TIKA-3329' of https://github.com/thammegowda/tika into main
     new b2f37ef  Forbidden API error fixes.

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../tika/language/translate/RTGTranslator.java     | 143 +++++++++++++++++++++
 .../org.apache.tika.language.translate.Translator  |   1 +
 ...eTranslatorTest.java => RTGTranslatorTest.java} |  47 ++-----
 3 files changed, 158 insertions(+), 33 deletions(-)
 create mode 100644 tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java
 copy tika-translate/src/test/java/org/apache/tika/language/translate/{GoogleTranslatorTest.java => RTGTranslatorTest.java} (62%)

[tika] 01/02: Backport Merge branch 'TIKA-3329' of https://github.com/thammegowda/tika into main

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mattmann pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git

commit 2c4ba60e2ce8f3388b92660f842fd63787ee4a5e
Author: Chris Mattmann <ch...@gmail.com>
AuthorDate: Sun May 2 08:10:51 2021 -0700

    Backport Merge branch 'TIKA-3329' of https://github.com/thammegowda/tika into main
---
 .../tika/language/translate/RTGTranslator.java     | 142 +++++++++++++++++++++
 .../org.apache.tika.language.translate.Translator  |   1 +
 .../tika/language/translate/RTGTranslatorTest.java |  62 +++++++++
 3 files changed, 205 insertions(+)

diff --git a/tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java b/tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java
new file mode 100644
index 0000000..ef366e2
--- /dev/null
+++ b/tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.tika.language.translate;
+
+import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider;
+import org.apache.cxf.jaxrs.client.WebClient;
+import org.apache.tika.exception.TikaException;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+
+/**
+ * <p>This translator is designed to work with a TCP-IP available
+ * RTG translation server, specifically the
+ * <a href="https://isi-nlp.github.io/rtg/#_rtg_serve">
+ * REST-based RTG server</a>.</p>
+ * To get Docker image:
+ *   https://hub.docker.com/repository/docker/tgowda/rtg-model <br/>
+ * <pre>
+ * {code
+ * # without GPU
+ *   docker run --rm -i -p 6060:6060 tgowda/rtg-model:500toEng-v1
+ * # Or, with GPU device 0
+ *   docker run --rm -i -p 6060:6060 --gpus '"device=0"' tgowda/rtg-model:500toEng-v1
+ * }
+ * </pre>
+ *
+ * <p>If you were to interact with the server via curl a request
+ * would look as follows</p>
+ *
+ * <pre>
+ * {code
+ * curl --data "source=Comment allez-vous?" \
+ *      --data "source=Bonne journée" \
+ *      http://localhost:6060/translate
+ * }
+ * </pre>
+ *
+ * RTG requires input to be pre-formatted into sentences, one per line,
+ * so this translation implementation takes care of that.
+ */
+public class RTGTranslator extends AbstractTranslator {
+
+    public static final String RTG_TRANSLATE_URL_BASE = "http://localhost:6060";
+    public static final String RTG_PROPS = "translator.rtg.properties";
+    private static final Logger LOG = LoggerFactory.getLogger(RTGTranslator.class);
+    private WebClient client;
+    private boolean isAvailable = false;
+
+    public RTGTranslator() {
+        String rtgBaseUrl = RTG_TRANSLATE_URL_BASE;
+        Properties config = new Properties();
+        try (InputStream stream = getClass().getClassLoader().getResourceAsStream(RTG_PROPS)){
+            if (stream != null){
+                config.load(stream);
+            }
+            rtgBaseUrl = config.getProperty("rtg.base.url", rtgBaseUrl);
+        } catch (IOException e) {
+            LOG.warn(e.getMessage(), e);
+        }
+        LOG.info("RTG base URL: " + rtgBaseUrl);
+        List<Object> providers = new ArrayList<>();
+        providers.add(new JacksonJsonProvider());
+        try {
+            this.client = WebClient.create(rtgBaseUrl, providers);
+            this.isAvailable = client.head().getStatus() == 200;
+        } catch (Exception e){
+            LOG.warn(e.getMessage(), e);
+            isAvailable = false;
+        }
+
+    }
+    @Override
+    public String translate(String text, String sourceLanguage, String targetLanguage)
+            throws TikaException, IOException {
+        return this.translate(text);
+    }
+
+    @Override
+    public String translate(String text, String targetLanguage)
+            throws TikaException, IOException {
+        return this.translate(text);
+    }
+
+    public String translate(String text) throws TikaException, IOException {
+        if (!this.isAvailable) {
+            return text;
+        }
+        Map<String, List<Object>> input = new HashMap<>();
+        input.put("source", Arrays.asList(text.split("(?<=(?<![A-Z])\\. )|\\n")));
+        Response response = client.path("translate")
+                .type(MediaType.APPLICATION_JSON)
+                .accept(MediaType.APPLICATION_JSON)
+                .post(input);
+        try (InputStreamReader reader = new InputStreamReader(
+                (InputStream) response.getEntity())) {
+            JSONParser parser = new JSONParser();
+            JSONObject obj = (JSONObject) parser.parse(reader);
+            List<String> sentences = (List<String>) obj.get("translation");
+            String output = String.join("\n", sentences);
+            return output;
+        } catch (ParseException e){
+            throw new IOException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public boolean isAvailable() {
+        return this.isAvailable;
+    }
+}
diff --git a/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator b/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator
index f3dcad4..154beca 100644
--- a/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator
+++ b/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator
@@ -18,3 +18,4 @@ org.apache.tika.language.translate.GoogleTranslator
 org.apache.tika.language.translate.Lingo24Translator
 org.apache.tika.language.translate.CachedTranslator
 org.apache.tika.language.translate.JoshuaNetworkTranslator
+org.apache.tika.language.translate.RTGTranslator
diff --git a/tika-translate/src/test/java/org/apache/tika/language/translate/RTGTranslatorTest.java b/tika-translate/src/test/java/org/apache/tika/language/translate/RTGTranslatorTest.java
new file mode 100644
index 0000000..0c18e0e
--- /dev/null
+++ b/tika-translate/src/test/java/org/apache/tika/language/translate/RTGTranslatorTest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.language.translate;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+
+/**
+ * Test harness for the {@link RTGTranslator}.
+ * 
+ */
+public class RTGTranslatorTest {
+
+	private RTGTranslator translator;
+
+
+	@Before
+	public void setUp() {
+		translator = new RTGTranslator();
+	}
+
+	@Test
+	public void testSimpleTranslate() {
+		String source = "hola señor";
+		String expected = "hello sir";
+
+		String result = null;
+		if (translator.isAvailable()) {
+			try {
+				result = translator.translate(source);
+				assertNotNull(result);
+				assertEquals("Result: [" + result
+						+ "]: not equal to expected: [" + expected + "]",
+						expected, result.toLowerCase());
+			} catch (Exception e) {
+				e.printStackTrace();
+				fail(e.getMessage());
+			}
+		}
+	}
+
+}

[tika] 02/02: Forbidden API error fixes.

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mattmann pushed a commit to branch branch_1x
in repository https://gitbox.apache.org/repos/asf/tika.git

commit b2f37ef2b330ec1954a4f96f02400ff6eaa792e7
Author: Chris Mattmann <ch...@gmail.com>
AuthorDate: Sat May 1 13:30:22 2021 -0700

    Forbidden API error fixes.
---
 .../main/java/org/apache/tika/language/translate/RTGTranslator.java | 3 ++-
 .../java/org/apache/tika/language/translate/RTGTranslatorTest.java  | 6 ++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java b/tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java
index ef366e2..8ff363a 100644
--- a/tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java
+++ b/tika-translate/src/main/java/org/apache/tika/language/translate/RTGTranslator.java
@@ -31,6 +31,7 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -124,7 +125,7 @@ public class RTGTranslator extends AbstractTranslator {
                 .accept(MediaType.APPLICATION_JSON)
                 .post(input);
         try (InputStreamReader reader = new InputStreamReader(
-                (InputStream) response.getEntity())) {
+							      (InputStream) response.getEntity(), Charset.defaultCharset())) {
             JSONParser parser = new JSONParser();
             JSONObject obj = (JSONObject) parser.parse(reader);
             List<String> sentences = (List<String>) obj.get("translation");
diff --git a/tika-translate/src/test/java/org/apache/tika/language/translate/RTGTranslatorTest.java b/tika-translate/src/test/java/org/apache/tika/language/translate/RTGTranslatorTest.java
index 0c18e0e..eec853e 100644
--- a/tika-translate/src/test/java/org/apache/tika/language/translate/RTGTranslatorTest.java
+++ b/tika-translate/src/test/java/org/apache/tika/language/translate/RTGTranslatorTest.java
@@ -20,6 +20,8 @@ package org.apache.tika.language.translate;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.Locale;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.fail;
@@ -42,7 +44,7 @@ public class RTGTranslatorTest {
 	@Test
 	public void testSimpleTranslate() {
 		String source = "hola señor";
-		String expected = "hello sir";
+		String expected = "hello, sir.";
 
 		String result = null;
 		if (translator.isAvailable()) {
@@ -51,7 +53,7 @@ public class RTGTranslatorTest {
 				assertNotNull(result);
 				assertEquals("Result: [" + result
 						+ "]: not equal to expected: [" + expected + "]",
-						expected, result.toLowerCase());
+					     expected, result.toLowerCase(Locale.getDefault()));
 			} catch (Exception e) {
 				e.printStackTrace();
 				fail(e.getMessage());