You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ma...@apache.org on 2014/06/06 00:49:22 UTC
svn commit: r1600787 - in /tika/trunk: ./
tika-core/src/main/java/org/apache/tika/
tika-core/src/main/java/org/apache/tika/config/
tika-core/src/main/java/org/apache/tika/language/translate/ tika-translate/
tika-translate/src/ tika-translate/src/main/ ...
Author: mattmann
Date: Thu Jun 5 22:49:21 2014
New Revision: 1600787
URL: http://svn.apache.org/r1600787
Log:
contribution for TIKA-1319: Translation module contributed by Tyler Palsulich.
Added:
tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/
tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/DefaultTranslator.java
tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/Translator.java
tika/trunk/tika-translate/
tika/trunk/tika-translate/pom.xml
tika/trunk/tika-translate/src/
tika/trunk/tika-translate/src/main/
tika/trunk/tika-translate/src/main/java/
tika/trunk/tika-translate/src/main/java/org/
tika/trunk/tika-translate/src/main/java/org/apache/
tika/trunk/tika-translate/src/main/java/org/apache/tika/
tika/trunk/tika-translate/src/main/java/org/apache/tika/language/
tika/trunk/tika-translate/src/main/java/org/apache/tika/language/translate/
tika/trunk/tika-translate/src/main/java/org/apache/tika/language/translate/MicrosoftTranslator.java
tika/trunk/tika-translate/src/main/resources/
tika/trunk/tika-translate/src/main/resources/META-INF/
tika/trunk/tika-translate/src/main/resources/META-INF/services/
tika/trunk/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator
tika/trunk/tika-translate/src/main/resources/org/
tika/trunk/tika-translate/src/main/resources/org/apache/
tika/trunk/tika-translate/src/main/resources/org/apache/tika/
tika/trunk/tika-translate/src/main/resources/org/apache/tika/language/
tika/trunk/tika-translate/src/main/resources/org/apache/tika/language/translator.microsoft.properties
tika/trunk/tika-translate/src/test/
tika/trunk/tika-translate/src/test/java/
tika/trunk/tika-translate/src/test/java/org/
tika/trunk/tika-translate/src/test/java/org/apache/
tika/trunk/tika-translate/src/test/java/org/apache/tika/
tika/trunk/tika-translate/src/test/java/org/apache/tika/language/
tika/trunk/tika-translate/src/test/java/org/apache/tika/language/translate/
tika/trunk/tika-translate/src/test/java/org/apache/tika/language/translate/MicrosoftTranslatorTest.java
Modified:
tika/trunk/CHANGES.txt
tika/trunk/pom.xml
tika/trunk/tika-core/src/main/java/org/apache/tika/Tika.java
tika/trunk/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
Modified: tika/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/tika/trunk/CHANGES.txt?rev=1600787&r1=1600786&r2=1600787&view=diff
==============================================================================
--- tika/trunk/CHANGES.txt (original)
+++ tika/trunk/CHANGES.txt Thu Jun 5 22:49:21 2014
@@ -1,5 +1,9 @@
Release 1.6 - ??/??/2014
+ * Added module and core Tika interface for translating text between
+ languages and added a default implementation that call's Microsoft's
+ translate service (TIKA-1319)
+
* Made RTFParser's list handling slightly more robust against corrupt
list metadata (TIKA-1305)
Modified: tika/trunk/pom.xml
URL: http://svn.apache.org/viewvc/tika/trunk/pom.xml?rev=1600787&r1=1600786&r2=1600787&view=diff
==============================================================================
--- tika/trunk/pom.xml (original)
+++ tika/trunk/pom.xml Thu Jun 5 22:49:21 2014
@@ -53,6 +53,7 @@
<module>tika-app</module>
<module>tika-bundle</module>
<module>tika-server</module>
+ <module>tika-translate</module>
</modules>
<build>
Modified: tika/trunk/tika-core/src/main/java/org/apache/tika/Tika.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/Tika.java?rev=1600787&r1=1600786&r2=1600787&view=diff
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/Tika.java (original)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/Tika.java Thu Jun 5 22:49:21 2014
@@ -27,7 +27,9 @@ import java.util.Properties;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.detect.Detector;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.io.IOUtils;
import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.language.translate.Translator;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
@@ -59,6 +61,11 @@ public class Tika {
private final Parser parser;
/**
+ * The Translator instance used by this facade.
+ */
+ private final Translator translator;
+
+ /**
* Maximum length of the strings returned by the parseToString methods.
* Used to prevent out of memory problems with huge input documents.
* The default setting is 100k characters.
@@ -66,7 +73,7 @@ public class Tika {
private int maxStringLength = 100 * 1000;
/**
- * Creates a Tika facade using the given detector and parser instances.
+ * Creates a Tika facade using the given detector and parser instances, but the default Translator.
*
* @since Apache Tika 0.8
* @param detector type detector
@@ -75,6 +82,21 @@ public class Tika {
public Tika(Detector detector, Parser parser) {
this.detector = detector;
this.parser = parser;
+ this.translator = TikaConfig.getDefaultConfig().getTranslator();
+ }
+
+ /**
+ * Creates a Tika facade using the given detector, parser, and translator instances.
+ *
+ * @since Apache Tika 1.6
+ * @param detector type detector
+ * @param parser document parser
+ * @param translator text translator
+ */
+ public Tika(Detector detector, Parser parser, Translator translator) {
+ this.detector = detector;
+ this.parser = parser;
+ this.translator = translator;
}
/**
@@ -83,7 +105,7 @@ public class Tika {
* @param config Tika configuration
*/
public Tika(TikaConfig config) {
- this(config.getDetector(), new AutoDetectParser(config));
+ this(config.getDetector(), new AutoDetectParser(config), config.getTranslator());
}
/**
@@ -94,8 +116,8 @@ public class Tika {
}
/**
- * Creates a Tika facade using the given detector instance and the
- * default parser configuration.
+ * Creates a Tika facade using the given detector instance, the
+ * default parser configuration, and the default Translator.
*
* @since Apache Tika 0.8
* @param detector type detector
@@ -289,6 +311,69 @@ public class Tika {
}
/**
+ * Translate the given text String to and from the given languages.
+ * @see org.apache.tika.language.translate.Translator
+ * @param text The text to translate.
+ * @param sourceLanguage The input text language (for example, "hi").
+ * @param targetLanguage The desired output language (for example, "fr").
+ * @return The translated text. If translation is unavailable (client keys not set), returns the same text back.
+ */
+ public String translate(String text, String sourceLanguage, String targetLanguage){
+ try {
+ return translator.translate(text, sourceLanguage, targetLanguage);
+ } catch (Exception e){
+ throw new IllegalStateException("Error translating data.", e);
+ }
+ }
+
+ /**
+ * Translate the given text String to the given language, attempting to auto-detect the source language.
+ * @see org.apache.tika.language.translate.Translator
+ * @param text The text to translate.
+ * @param targetLanguage The desired output language (for example, "en").
+ * @return The translated text. If translation is unavailable (client keys not set), returns the same text back.
+ */
+ public String translate(String text, String targetLanguage){
+ try {
+ return translator.translate(text, targetLanguage);
+ } catch (Exception e){
+ throw new IllegalStateException("Error translating data.", e);
+ }
+ }
+
+ /**
+ * Translate the given text InputStream to and from the given languages.
+ * @see org.apache.tika.language.translate.Translator
+ * @param text The text to translate.
+ * @param sourceLanguage The input text language (for example, "hi").
+ * @param targetLanguage The desired output language (for example, "fr").
+ * @return The translated text. If translation is unavailable (client keys not set), returns the same text back.
+ */
+ public String translate(InputStream text, String sourceLanguage, String targetLanguage){
+ try {
+ return translator.translate(IOUtils.toString(text), sourceLanguage, targetLanguage);
+ } catch (Exception e){
+ throw new IllegalStateException("Error translating data.", e);
+ }
+ }
+
+ /**
+ * Translate the given text InputStream to the given language, attempting to auto-detect the source language.
+ * This does not close the stream, so the caller has the responsibility of closing it.
+ * @see org.apache.tika.language.translate.Translator
+ * @param text The text to translate.
+ * @param targetLanguage The desired output language (for example, "en").
+ * @return The translated text. If translation is unavailable (client keys not set), returns the same text back.
+ */
+ public String translate(InputStream text, String targetLanguage){
+ try {
+ return translator.translate(IOUtils.toString(text), targetLanguage);
+ } catch (Exception e){
+ throw new IllegalStateException("Error translating data.", e);
+ }
+ }
+
+ /**
* Parses the given document and returns the extracted text content.
* Input metadata like a file name or a content type hint can be passed
* in the given metadata instance. Metadata information extracted from
@@ -535,6 +620,16 @@ public class Tika {
return detector;
}
+ /**
+ * Returns the translator instance used by this facade.
+ *
+ * @since Tika 1.6
+ * @return translator instance
+ */
+ public Translator getTranslator() {
+ return translator;
+ }
+
//--------------------------------------------------------------< Object >
public String toString() {
Modified: tika/trunk/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java?rev=1600787&r1=1600786&r2=1600787&view=diff
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java (original)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/config/TikaConfig.java Thu Jun 5 22:49:21 2014
@@ -35,6 +35,8 @@ import org.apache.tika.detect.CompositeD
import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.exception.TikaException;
+import org.apache.tika.language.translate.DefaultTranslator;
+import org.apache.tika.language.translate.Translator;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MediaTypeRegistry;
import org.apache.tika.mime.MimeTypeException;
@@ -70,8 +72,12 @@ public class TikaConfig {
return new DefaultParser(types.getMediaTypeRegistry(), loader);
}
+ private static Translator getDefaultTranslator(ServiceLoader loader) {
+ return new DefaultTranslator(loader);
+ }
private final CompositeParser parser;
private final Detector detector;
+ private final Translator translator;
private final MimeTypes mimeTypes;
@@ -118,6 +124,7 @@ public class TikaConfig {
this.mimeTypes = typesFromDomElement(element);
this.detector = detectorFromDomElement(element, mimeTypes, loader);
this.parser = parserFromDomElement(element, mimeTypes, loader);
+ this.translator = translatorFromDomElement(element, loader);
}
/**
@@ -138,6 +145,7 @@ public class TikaConfig {
this.mimeTypes = getDefaultMimeTypes(loader);
this.detector = getDefaultDetector(mimeTypes, serviceLoader);
this.parser = getDefaultParser(mimeTypes, serviceLoader);
+ this.translator = getDefaultTranslator(serviceLoader);
}
/**
@@ -169,6 +177,7 @@ public class TikaConfig {
this.mimeTypes = getDefaultMimeTypes(ServiceLoader.getContextClassLoader());
this.parser = getDefaultParser(mimeTypes, loader);
this.detector = getDefaultDetector(mimeTypes, loader);
+ this.translator = getDefaultTranslator(loader);
} else {
// Locate the given configuration file
InputStream stream = null;
@@ -198,6 +207,7 @@ public class TikaConfig {
parserFromDomElement(element, mimeTypes, loader);
this.detector =
detectorFromDomElement(element, mimeTypes, loader);
+ this.translator = translatorFromDomElement(element, loader);
} catch (SAXException e) {
throw new TikaException(
"Specified Tika configuration has syntax errors: "
@@ -248,6 +258,15 @@ public class TikaConfig {
return detector;
}
+ /**
+ * Returns the configured translator instance.
+ *
+ * @return configured translator
+ */
+ public Translator getTranslator() {
+ return translator;
+ }
+
public MimeTypes getMimeRepository(){
return mimeTypes;
}
@@ -392,4 +411,35 @@ public class TikaConfig {
return new CompositeDetector(registry, detectors);
}
}
+
+ private static Translator translatorFromDomElement(
+ Element element, ServiceLoader loader)
+ throws TikaException, IOException {
+ List<Translator> translators = new ArrayList<Translator>();
+ NodeList nodes = element.getElementsByTagName("translator");
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element node = (Element) nodes.item(i);
+ String name = node.getAttribute("class");
+
+ try {
+ Class<? extends Translator> translatorClass =
+ loader.getServiceClass(Translator.class, name);
+ translators.add(translatorClass.newInstance());
+ } catch (ClassNotFoundException e) {
+ throw new TikaException(
+ "Unable to find a translator class: " + name, e);
+ } catch (IllegalAccessException e) {
+ throw new TikaException(
+ "Unable to access a translator class: " + name, e);
+ } catch (InstantiationException e) {
+ throw new TikaException(
+ "Unable to instantiate a translator class: " + name, e);
+ }
+ }
+ if (translators.isEmpty()) {
+ return getDefaultTranslator(loader);
+ } else {
+ return translators.get(0);
+ }
+ }
}
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/DefaultTranslator.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/DefaultTranslator.java?rev=1600787&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/DefaultTranslator.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/DefaultTranslator.java Thu Jun 5 22:49:21 2014
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.language.translate;
+
+import org.apache.tika.config.ServiceLoader;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class DefaultTranslator implements Translator{
+ private transient final ServiceLoader loader;
+
+ public DefaultTranslator(ServiceLoader loader) {
+ this.loader = loader;
+ }
+
+ /**
+ * Finds all statically loadable translators and sort the list by name,
+ * rather than discovery order.
+ *
+ * @param loader service loader
+ * @return ordered list of statically loadable parsers
+ */
+ private static List<Translator> getDefaultTranslators(ServiceLoader loader) {
+ List<Translator> translators = loader.loadStaticServiceProviders(Translator.class);
+ Collections.sort(translators, new Comparator<Translator>() {
+ public int compare(Translator t1, Translator t2) {
+ String n1 = t1.getClass().getName();
+ String n2 = t2.getClass().getName();
+ boolean tika1 = n1.startsWith("org.apache.tika.");
+ boolean tika2 = n2.startsWith("org.apache.tika.");
+ if (tika1 == tika2) {
+ return n1.compareTo(n2);
+ } else if (tika1) {
+ return -1;
+ } else {
+ return 1;
+ }
+ }
+ });
+ return translators;
+ }
+
+ public String translate(String text, String sourceLanguage, String targetLanguage) throws Exception {
+ return getDefaultTranslators(loader).get(0).translate(text, sourceLanguage, targetLanguage);
+ }
+
+ public String translate(String text, String targetLanguage) throws Exception {
+ return getDefaultTranslators(loader).get(0).translate(text, targetLanguage);
+ }
+
+ public boolean isAvailable() {
+ return getDefaultTranslators(loader).get(0).isAvailable();
+ }
+
+}
Added: tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/Translator.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/Translator.java?rev=1600787&view=auto
==============================================================================
--- tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/Translator.java (added)
+++ tika/trunk/tika-core/src/main/java/org/apache/tika/language/translate/Translator.java Thu Jun 5 22:49:21 2014
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.language.translate;
+
+/**
+ * Interface for Translator services.
+ * @since Tika 1.6
+ */
+public interface Translator {
+ /**
+ * Translate text between given languages. The following languages are supported:
+ * Arabic("ar"), Bulgarian("bg"), Catalan("ca"), Chinese-Simplified("zh-CHS"), Chinese-Traditional("zh-CHT"),
+ * Czech("cs"), Danish("da"), Dutch("nl"), English("en"), Estonian("et"), Innish("fi"), French("fr"), German("de"),
+ * Greek("el"), Haitian-Creole("ht"), Hebrew("he"), Hindi("hi"), Hmong-Daw("mww"), Hungarian("hu"),
+ * Indonesian("id"), Italian("it"), Japanese("ja"), Korean("ko"), Latvian("lv"), Lithuanian("lt"), Malay("ms"),
+ * Norwegian("no"), Persian("fa"), Polish("pl"), Portuguese("pt"), Romanian("ro"), Russian("ru"), Slovak("sk"),
+ * Slovenian("sl"), Spanish("es"), Swedish("sv"), Thai("th"), Turkish("tr"), Ukranian("uk"), Urdu("ur"),
+ * Vietnemese("vi").
+ * @param text The text to translate.
+ * @param sourceLanguage The input text language (for example, "en").
+ * @param targetLanguage The desired language to translate to (for example, "fr").
+ * @return The translation result. If translation is unavailable, returns the same text back.
+ * @throws Exception When there is an error with the API call.
+ * @since Tika 1.6
+ */
+ public String translate(String text, String sourceLanguage, String targetLanguage) throws Exception;
+
+ /**
+ * Translate text to the given language. This method attempts to auto-detect the source language of the text.
+ * The following languages are supported:
+ * Arabic("ar"), Bulgarian("bg"), Catalan("ca"), Chinese-Simplified("zh-CHS"), Chinese-Traditional("zh-CHT"),
+ * Czech("cs"), Danish("da"), Dutch("nl"), English("en"), Estonian("et"), Innish("fi"), French("fr"), German("de"),
+ * Greek("el"), Haitian-Creole("ht"), Hebrew("he"), Hindi("hi"), Hmong-Daw("mww"), Hungarian("hu"),
+ * Indonesian("id"), Italian("it"), Japanese("ja"), Korean("ko"), Latvian("lv"), Lithuanian("lt"), Malay("ms"),
+ * Norwegian("no"), Persian("fa"), Polish("pl"), Portuguese("pt"), Romanian("ro"), Russian("ru"), Slovak("sk"),
+ * Slovenian("sl"), Spanish("es"), Swedish("sv"), Thai("th"), Turkish("tr"), Ukranian("uk"), Urdu("ur"),
+ * Vietnemese("vi").
+ * @param text The text to translate.
+ * @param targetLanguage The desired language to translate to (for example, "hi").
+ * @return The translation result. If translation is unavailable, returns the same text back.
+ * @throws Exception When there is an error with the API call.
+ * @since Tika 1.6
+ */
+ public String translate(String text, String targetLanguage) throws Exception;
+
+ /**
+ * @return true if this Translator is probably able to translate right now.
+ * @since Tika 1.6
+ */
+ public boolean isAvailable();
+}
Added: tika/trunk/tika-translate/pom.xml
URL: http://svn.apache.org/viewvc/tika/trunk/tika-translate/pom.xml?rev=1600787&view=auto
==============================================================================
--- tika/trunk/tika-translate/pom.xml (added)
+++ tika/trunk/tika-translate/pom.xml Thu Jun 5 22:49:21 2014
@@ -0,0 +1,151 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-parent</artifactId>
+ <version>1.6-SNAPSHOT</version>
+ <relativePath>../tika-parent/pom.xml</relativePath>
+ </parent>
+
+ <artifactId>tika-translate</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Tika translate</name>
+ <url>http://tika.apache.org/</url>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.tika</groupId>
+ <artifactId>tika-core</artifactId>
+ <version>1.6-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>com.memetix</groupId>
+ <artifactId>microsoft-translator-java-api</artifactId>
+ <version>0.6.2</version>
+ <type>jar</type>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ <version>4.11</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <extensions>true</extensions>
+ <configuration>
+ <instructions>
+ <Bundle-DocURL>${project.url}</Bundle-DocURL>
+ <Bundle-Activator>
+ org.apache.tika.parser.internal.Activator
+ </Bundle-Activator>
+ <Import-Package>
+ org.w3c.dom,
+ org.apache.tika.*,
+ *;resolution:=optional
+ </Import-Package>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/main/java/org/apache/tika/parser/txt/Charset*.java</exclude>
+ <exclude>src/test/resources/test-documents/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+
+ <pluginManagement>
+ <plugins>
+ <!-- This plugin's configuration is used to store Eclipse m2e -->
+ <!-- settings only. It has no influence on the Maven build itself. -->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-scr-plugin</artifactId>
+ <versionRange>[1.7.2,)</versionRange>
+ <goals>
+ <goal>scr</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute />
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+ <description>This is the translate Apache Tika⢠toolkit. Translator implementations may depend on web services. </description>
+ <organization>
+ <name>The Apache Software Foundation</name>
+ <url>http://www.apache.org</url>
+ </organization>
+ <scm>
+ <url>http://svn.apache.org/viewvc/tika/trunk/translate</url>
+ <connection>scm:svn:http://svn.apache.org/repos/asf/tika/trunk/translate</connection>
+ <developerConnection>scm:svn:https://svn.apache.org/repos/asf/tika/trunk/translate</developerConnection>
+ </scm>
+ <issueManagement>
+ <system>JIRA</system>
+ <url>https://issues.apache.org/jira/browse/TIKA</url>
+ </issueManagement>
+ <ciManagement>
+ <system>Jenkins</system>
+ <url>https://builds.apache.org/job/Tika-trunk/</url>
+ </ciManagement>
+</project>
Added: tika/trunk/tika-translate/src/main/java/org/apache/tika/language/translate/MicrosoftTranslator.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-translate/src/main/java/org/apache/tika/language/translate/MicrosoftTranslator.java?rev=1600787&view=auto
==============================================================================
--- tika/trunk/tika-translate/src/main/java/org/apache/tika/language/translate/MicrosoftTranslator.java (added)
+++ tika/trunk/tika-translate/src/main/java/org/apache/tika/language/translate/MicrosoftTranslator.java Thu Jun 5 22:49:21 2014
@@ -0,0 +1,97 @@
+package org.apache.tika.language.translate;
+
+import com.memetix.mst.language.Language;
+import com.memetix.mst.translate.Translate;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
+/**
+ * Wrapper class to access the Windows translation service. This class uses the com.memetix.mst
+ * package as a wrapper for the API calls.
+ * @since Tika 1.6
+ */
+public class MicrosoftTranslator implements Translator {
+
+ boolean available; // Flag for whether or not translation is available.
+ String clientId, clientSecret; // Keys used for the API calls.
+
+ public static final String PROPERTIES_FILE = "translator.microsoft.properties";
+ public static final String ID_PROPERTY = "translator.client-id";
+ public static final String SECRET_PROPERTY = "translator.client-secret";
+ public static final String DEFAULT_ID = "dummy-id";
+ public static final String DEFAULT_SECRET = "dummy-secret";
+
+ /**
+ * Create a new MicrosoftTranslator with the client keys specified in
+ * resources/org/apache/tika/language/translate/translator.microsoft.properties. Silently becomes unavailable
+ * when client keys are unavailable. translator.microsoft.client-id and translator.client-secret must be set
+ * in translator.microsoft.properties for translation to work.
+ * @since Tika 1.6
+ */
+ public MicrosoftTranslator() {
+ Properties props = new Properties();
+ InputStream stream;
+ stream = Translator.class.getResourceAsStream(PROPERTIES_FILE);
+ try {
+ if(stream != null) {
+ props.load(stream);
+ clientId = props.getProperty(ID_PROPERTY);
+ clientSecret = props.getProperty(SECRET_PROPERTY);
+ if (!clientId.equals(DEFAULT_ID) && !clientSecret.equals(DEFAULT_SECRET)) available = true;
+ }
+ } catch (IOException e) {
+ // Error with properties file. Translation will not work.
+ available = false;
+ }
+ }
+
+ /**
+ * Use the Microsoft service to translate the given text from the given source language to the given target.
+ * You must set the client keys in translator.microsoft.properties.
+ *
+ * @param text The text to translate.
+ * @param sourceLanguage The input text language (for example, "en").
+ * @param targetLanguage The desired language to translate to (for example, "fr").
+ * @return The translated text. If translation is unavailable, returns the unchanged text.
+ * @throws Exception
+ * @see org.apache.tika.language.translate.Translator
+ * @since Tika 1.6
+ */
+ public String translate(String text, String sourceLanguage, String targetLanguage) throws Exception {
+ if (!available) return text;
+ Language source = Language.fromString(sourceLanguage);
+ Language target = Language.fromString(targetLanguage);
+ Translate.setClientId(clientId);
+ Translate.setClientSecret(clientSecret);
+ return Translate.execute(text, source, target);
+ }
+
+ /**
+ * Use the Microsoft service to translate the given text to the given target language. The source language
+ * is automatically detected by Microsoft. You must set the client keys in translator.microsoft.properties.
+ * @param text The text to translate.
+ * @param targetLanguage The desired language to translate to (for example, "hi").
+ * @return The translated text. If translation is unavailable, returns the unchanged text.
+ * @throws Exception
+ * @see org.apache.tika.language.translate.Translator
+ * @since Tika 1.6
+ */
+ public String translate(String text, String targetLanguage) throws Exception {
+ if (!available) return text;
+ Language target = Language.fromString(targetLanguage);
+ Translate.setClientId(clientId);
+ Translate.setClientSecret(clientSecret);
+ return Translate.execute(text, target);
+ }
+
+ /**
+ * Check whether this instance has a working property file and its keys are not the defaults.
+ * This is not guaranteed to work, since keys may be incorrect or the webservice may be down.
+ * @return whether translation will probably work.
+ */
+ public boolean isAvailable(){
+ return available;
+ }
+}
Added: tika/trunk/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator
URL: http://svn.apache.org/viewvc/tika/trunk/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator?rev=1600787&view=auto
==============================================================================
--- tika/trunk/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator (added)
+++ tika/trunk/tika-translate/src/main/resources/META-INF/services/org.apache.tika.language.translate.Translator Thu Jun 5 22:49:21 2014
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.tika.language.translate.MicrosoftTranslator
\ No newline at end of file
Added: tika/trunk/tika-translate/src/main/resources/org/apache/tika/language/translator.microsoft.properties
URL: http://svn.apache.org/viewvc/tika/trunk/tika-translate/src/main/resources/org/apache/tika/language/translator.microsoft.properties?rev=1600787&view=auto
==============================================================================
--- tika/trunk/tika-translate/src/main/resources/org/apache/tika/language/translator.microsoft.properties (added)
+++ tika/trunk/tika-translate/src/main/resources/org/apache/tika/language/translator.microsoft.properties Thu Jun 5 22:49:21 2014
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Must set the client keys in this file to use translation. Please see
+# https://code.google.com/p/microsoft-translator-java-api/ and
+# http://msdn.microsoft.com/en-us/library/hh454950.aspx for help with
+# getting these keys. As of now (6/2014) 2,000,000 characters/month
+# are free.
+#
+# To use the Microsoft translation service, you <em>must</em> set these
+# keys, as described in MicrosoftTranslator. If you do not want translation
+# please set these values to "dummy-id" and "dummy-secret".
+
+translator.client-id=dummy-id
+translator.client-secret=dummy-secret
Added: tika/trunk/tika-translate/src/test/java/org/apache/tika/language/translate/MicrosoftTranslatorTest.java
URL: http://svn.apache.org/viewvc/tika/trunk/tika-translate/src/test/java/org/apache/tika/language/translate/MicrosoftTranslatorTest.java?rev=1600787&view=auto
==============================================================================
--- tika/trunk/tika-translate/src/test/java/org/apache/tika/language/translate/MicrosoftTranslatorTest.java (added)
+++ tika/trunk/tika-translate/src/test/java/org/apache/tika/language/translate/MicrosoftTranslatorTest.java Thu Jun 5 22:49:21 2014
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.language.translate;
+
+import org.apache.tika.Tika;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test cases for the {@link MicrosoftTranslator} class.
+ */
+public class MicrosoftTranslatorTest {
+ Tika tika;
+ @Before
+ public void setUp() {
+ tika = new Tika();
+ }
+
+ @Test
+ public void testSimpleTranslate() throws Exception {
+ String source = "hello";
+ String expected = "salut";
+ String translated = tika.translate(source, "en", "fr");
+ System.err.println(tika.getTranslator().isAvailable());
+ if (tika.getTranslator().isAvailable()) assertTrue("Translate " + source + " to " + expected + " (was " + translated + ")",
+ expected.equalsIgnoreCase(translated));
+ }
+
+ @Test
+ public void testSimpleDetectTranslate() throws Exception {
+ String source = "hello";
+ String expected = "salut";
+ String translated = tika.translate(source, "fr");
+ System.err.println(tika.getTranslator().isAvailable());
+ if (tika.getTranslator().isAvailable()) assertTrue("Translate " + source + " to " + expected + " (was " + translated + ")",
+ expected.equalsIgnoreCase(translated));
+ }
+
+}