(incubator-stormcrawler) branch main updated: #1248 - Use pre-compiled patterns for mime type matching in TikaParser (#1249)

[jn...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jnioche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git


The following commit(s) were added to refs/heads/main by this push:
     new ef0899ea #1248 - Use pre-compiled patterns for mime type matching in TikaParser (#1249)
ef0899ea is described below

commit ef0899ea2c17873c972670b83f4853311f61e4fa
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Fri Jul 5 16:23:49 2024 +0200

    #1248 - Use pre-compiled patterns for mime type matching in TikaParser (#1249)
    
    Co-authored-by: Julien Nioche <ju...@digitalpebble.com>
---
 .../java/org/apache/stormcrawler/tika/ParserBolt.java | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
index 1239679e..94204039 100644
--- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
+++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java
@@ -27,6 +27,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 import org.apache.commons.lang.StringUtils;
 import org.apache.html.dom.HTMLDocumentImpl;
 import org.apache.storm.metric.api.MultiCountMetric;
@@ -92,7 +93,7 @@ public class ParserBolt extends BaseRichBolt {
     private boolean emitOutlinks = true;
 
     /** regular expressions to apply to the mime-type * */
-    private List<String> mimeTypeWhiteList = new LinkedList<>();
+    private List<Pattern> mimeTypeWhiteList = new LinkedList<>();
 
     private String protocolMDprefix;
 
@@ -125,7 +126,15 @@ public class ParserBolt extends BaseRichBolt {
             throw e;
         }
 
-        mimeTypeWhiteList = ConfUtils.loadListFromConf("parser.mimetype.whitelist", conf);
+        final List<String> mimeTypeWhiteListStrings =
+                ConfUtils.loadListFromConf("parser.mimetype.whitelist", conf);
+        for (String mt : mimeTypeWhiteListStrings) {
+            try {
+                this.mimeTypeWhiteList.add(Pattern.compile(mt));
+            } catch (RuntimeException e) {
+                LOG.warn("Failed to compile whitelist regex: {}", mt);
+            }
+        }
 
         protocolMDprefix = ConfUtils.getString(conf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "");
 
@@ -149,7 +158,7 @@ public class ParserBolt extends BaseRichBolt {
         Metadata metadata = (Metadata) tuple.getValueByField("metadata");
 
         // check that the mimetype is in the whitelist
-        if (mimeTypeWhiteList.size() > 0) {
+        if (!mimeTypeWhiteList.isEmpty()) {
             boolean mt_match = false;
             // see if a mimetype was guessed in JSOUPBolt
             String mimeType = metadata.getFirstValue("parse.Content-Type");
@@ -158,8 +167,8 @@ public class ParserBolt extends BaseRichBolt {
                 mimeType = metadata.getFirstValue(HttpHeaders.CONTENT_TYPE, this.protocolMDprefix);
             }
             if (mimeType != null) {
-                for (String mt : mimeTypeWhiteList) {
-                    if (mimeType.matches(mt)) {
+                for (Pattern mt : mimeTypeWhiteList) {
+                    if (mt.matcher(mimeType).matches()) {
                         mt_match = true;
                         break;
                     }