You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2018/11/09 18:17:39 UTC
[tika] 02/02: TIKA-2777 -- improve inefficient regex performance in
Optimaize in tika-eval
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tika.git
commit b899ed2084da5e55e6f6f67f9ed6320942e4fd6d
Author: TALLISON <ta...@apache.org>
AuthorDate: Fri Nov 9 13:15:03 2018 -0500
TIKA-2777 -- improve inefficient regex performance in Optimaize in tika-eval
---
CHANGES.txt | 1 +
.../org/apache/tika/eval/AbstractProfiler.java | 2 +
.../apache/tika/eval/util/LanguageIDWrapper.java | 44 ++++++++++++++++++++--
.../org/apache/tika/eval/io/ExtractReaderTest.java | 1 -
.../org/apache/tika/eval/util/LanguageIdTest.java | 37 ++++++++++++++++++
.../tika/langdetect/OptimaizeLangDetectorTest.java | 14 ++++++-
6 files changed, 93 insertions(+), 6 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 1bcde7f..e949016 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 2.0.0 - ???
Release 1.20 - ???
+ * Improve language id efficiency in tika-eval (TIKA-2777).
* Upgrade sqlite "provided" dependency to 3.25.2 (TIKA-2773).
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
index 322a1f6..0b41acb 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/AbstractProfiler.java
@@ -177,6 +177,7 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
super(fileQueue);
this.writer = writer;
langIder = new LanguageIDWrapper();
+ LanguageIDWrapper.setMaxTextLength(maxContentLengthForLangId);
initAnalyzersAndTokenCounter(maxTokens);
}
@@ -205,6 +206,7 @@ public abstract class AbstractProfiler extends FileResourceConsumer {
*/
public void setMaxContentLengthForLangId(int maxContentLengthForLangId) {
this.maxContentLengthForLangId = maxContentLengthForLangId;
+ LanguageIDWrapper.setMaxTextLength(maxContentLengthForLangId);
}
/**
diff --git a/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java b/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java
index 59d032a..2f3d42b 100644
--- a/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java
+++ b/tika-eval/src/main/java/org/apache/tika/eval/util/LanguageIDWrapper.java
@@ -18,7 +18,9 @@ package org.apache.tika.eval.util;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Pattern;
import com.google.common.base.Optional;
import com.optimaize.langdetect.DetectedLanguage;
@@ -28,7 +30,10 @@ import com.optimaize.langdetect.i18n.LdLocale;
import com.optimaize.langdetect.ngram.NgramExtractors;
import com.optimaize.langdetect.profiles.LanguageProfile;
import com.optimaize.langdetect.profiles.LanguageProfileReader;
-import com.optimaize.langdetect.text.CommonTextObjectFactories;
+import com.optimaize.langdetect.text.MultiTextFilter;
+import com.optimaize.langdetect.text.RemoveMinorityScriptsTextFilter;
+import com.optimaize.langdetect.text.TextFilter;
+import com.optimaize.langdetect.text.TextObject;
import com.optimaize.langdetect.text.TextObjectFactory;
@@ -37,13 +42,15 @@ public class LanguageIDWrapper {
static LanguageDetector detector;
static TextObjectFactory textObjectFactory;
+ static int MAX_TEXT_LENGTH = 50000;
+
public static void loadBuiltInModels() throws IOException {
languageProfiles = new LanguageProfileReader().readAllBuiltIn();
detector = LanguageDetectorBuilder.create(NgramExtractors.standard())
.withProfiles(languageProfiles)
.build();
- textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
+ textObjectFactory = buildTextObjectFactory();
}
public static void loadModels(Path path) throws IOException {
@@ -52,9 +59,15 @@ public class LanguageIDWrapper {
detector = LanguageDetectorBuilder.create(NgramExtractors.standard())
.withProfiles(languageProfiles)
.build();
- textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
+ textObjectFactory = buildTextObjectFactory();
}
+ private static TextObjectFactory buildTextObjectFactory() {
+ List<TextFilter> textFilters = new ArrayList<>();
+ textFilters.add(TikasUrlTextFilter.getInstance());
+ textFilters.add(RemoveMinorityScriptsTextFilter.forThreshold(0.3));
+ return new TextObjectFactory(new MultiTextFilter(textFilters), MAX_TEXT_LENGTH);
+ }
public static Optional<LdLocale> detect(String s) {
@@ -62,8 +75,31 @@ public class LanguageIDWrapper {
}
public static List<DetectedLanguage> getProbabilities(String s) {
+ TextObject textObject = textObjectFactory.forText(s);
+ return detector.getProbabilities(textObject);
+ }
- return detector.getProbabilities(textObjectFactory.forText(s));
+ public static void setMaxTextLength(int maxTextLength) {
+ MAX_TEXT_LENGTH = maxTextLength;
}
+ private static class TikasUrlTextFilter implements TextFilter {
+ //use this custom copy/paste of optimaize to avoid long, long hang with mail_regex
+ //TIKA-2777
+ private static final Pattern URL_REGEX = Pattern.compile("https?://[-_.?&~;+=/#0-9A-Za-z]{10,10000}");
+ private static final Pattern MAIL_REGEX = Pattern.compile("[-_.0-9A-Za-z]{1,100}@[-_0-9A-Za-z]{1,100}[-_.0-9A-Za-z]{1,100}");
+ private static final TikasUrlTextFilter INSTANCE = new TikasUrlTextFilter();
+
+ public static TikasUrlTextFilter getInstance() {
+ return INSTANCE;
+ }
+
+ private TikasUrlTextFilter() {
+ }
+
+ public String filter(CharSequence text) {
+ String modified = URL_REGEX.matcher(text).replaceAll(" ");
+ return MAIL_REGEX.matcher(modified).replaceAll(" ");
+ }
+ }
}
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
index 47d5934..ff34199 100644
--- a/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
+++ b/tika-eval/src/test/java/org/apache/tika/eval/io/ExtractReaderTest.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
-import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.sax.AbstractRecursiveParserWrapperHandler;
import org.junit.Before;
import org.junit.Test;
diff --git a/tika-eval/src/test/java/org/apache/tika/eval/util/LanguageIdTest.java b/tika-eval/src/test/java/org/apache/tika/eval/util/LanguageIdTest.java
new file mode 100644
index 0000000..bb4cd9c
--- /dev/null
+++ b/tika-eval/src/test/java/org/apache/tika/eval/util/LanguageIdTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.eval.util;
+
+import com.google.common.base.Optional;
+import com.optimaize.langdetect.i18n.LdLocale;
+import org.apache.tika.eval.util.LanguageIDWrapper;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LanguageIdTest {
+ @Test(timeout = 10000)
+ public void testDefenseAgainstBadRegexInOptimaize() throws Exception {
+ //TIKA-2777
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 50000; i++) {
+ sb.append("a");
+ }
+ LanguageIDWrapper.loadBuiltInModels();
+ Optional<LdLocale> optional = LanguageIDWrapper.detect(sb.toString());
+ Assert.assertEquals("so", optional.get().getLanguage());
+ }
+}
diff --git a/tika-langdetect/src/test/java/org/apache/tika/langdetect/OptimaizeLangDetectorTest.java b/tika-langdetect/src/test/java/org/apache/tika/langdetect/OptimaizeLangDetectorTest.java
index 3bf6f8c..dbcaf26 100644
--- a/tika-langdetect/src/test/java/org/apache/tika/langdetect/OptimaizeLangDetectorTest.java
+++ b/tika-langdetect/src/test/java/org/apache/tika/langdetect/OptimaizeLangDetectorTest.java
@@ -260,6 +260,18 @@ public class OptimaizeLangDetectorTest extends LanguageDetectorTest {
return result;
}
-
+
+ @Test(timeout = 5000)
+ public void testOptimaizeRegexBug() throws Exception {
+ //confirm TIKA-2777 doesn't affect langdetect's Optimaize
+ LanguageDetector detector = new OptimaizeLangDetector()
+ .setShortText(false)
+ .loadModels();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 50000; i++) {
+ sb.append('a');
+ }
+ detector.detect(sb.toString());
+ }
}