You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2024/07/30 15:41:29 UTC

(tika) branch TIKA-4288 created (now 480dfb743)

This is an automated email from the ASF dual-hosted git repository.

tallison pushed a change to branch TIKA-4288
in repository https://gitbox.apache.org/repos/asf/tika.git


      at 480dfb743 TIKA-4288 -- users should be able to configure the metadata filter per parse via the ParseContext on the ParseEmitTuple with PipesServer

This branch includes the following new commits:

     new 480dfb743 TIKA-4288 -- users should be able to configure the metadata filter per parse via the ParseContext on the ParseEmitTuple with PipesServer

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(tika) 01/01: TIKA-4288 -- users should be able to configure the metadata filter per parse via the ParseContext on the ParseEmitTuple with PipesServer

Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tallison pushed a commit to branch TIKA-4288
in repository https://gitbox.apache.org/repos/asf/tika.git

commit 480dfb74372e6f57d5df53ad9a4986dce1e4680b
Author: tallison <ta...@apache.org>
AuthorDate: Tue Jul 30 11:41:12 2024 -0400

    TIKA-4288 -- users should be able to configure the metadata filter per parse via the ParseContext on the ParseEmitTuple with PipesServer
---
 .../java/org/apache/tika/pipes/PipesServer.java    | 13 +++++++++----
 .../org/apache/tika/pipes/PipesClientTest.java     | 22 ++++++++++++++++++++--
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
index 802545380..54d2b439c 100644
--- a/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
+++ b/tika-core/src/main/java/org/apache/tika/pipes/PipesServer.java
@@ -57,6 +57,7 @@ import org.apache.tika.io.TemporaryResources;
 import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
+import org.apache.tika.metadata.filter.MetadataFilter;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.DigestingParser;
@@ -398,8 +399,12 @@ public class PipesServer implements Runnable {
     private void emitParseData(FetchEmitTuple t, MetadataListAndEmbeddedBytes parseData) {
         long start = System.currentTimeMillis();
         String stack = getContainerStacktrace(t, parseData.getMetadataList());
-        //we need to apply this after we pull out the stacktrace
-        filterMetadata(parseData.getMetadataList());
+        //we need to apply the metadata filter after we pull out the stacktrace
+        MetadataFilter filter = t.getParseContext().get(MetadataFilter.class);
+        if (filter == null) {
+            filter = tikaConfig.getMetadataFilter();
+        }
+        filterMetadata(filter, parseData.getMetadataList());
         ParseContext parseContext = t.getParseContext();
         FetchEmitTuple.ON_PARSE_EXCEPTION onParseException = t.getOnParseException();
         EmbeddedDocumentBytesConfig embeddedDocumentBytesConfig = parseContext.get(EmbeddedDocumentBytesConfig.class);
@@ -432,10 +437,10 @@ public class PipesServer implements Runnable {
         }
     }
 
-    private void filterMetadata(List<Metadata> metadataList) {
+    private void filterMetadata(MetadataFilter metadataFilter, List<Metadata> metadataList) {
         for (Metadata m : metadataList) {
             try {
-                tikaConfig.getMetadataFilter().filter(m);
+                metadataFilter.filter(m);
             } catch (TikaException e) {
                 LOG.warn("failed to filter metadata", e);
             }
diff --git a/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java b/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java
index 01cc86c0a..13b0dc312 100644
--- a/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java
+++ b/tika-core/src/test/java/org/apache/tika/pipes/PipesClientTest.java
@@ -19,6 +19,7 @@ package org.apache.tika.pipes;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
 import javax.xml.parsers.ParserConfigurationException;
 
 import org.junit.jupiter.api.Assertions;
@@ -28,11 +29,14 @@ import org.xml.sax.SAXException;
 
 import org.apache.tika.exception.TikaConfigException;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.filter.CompositeMetadataFilter;
+import org.apache.tika.metadata.filter.MetadataFilter;
+import org.apache.tika.metadata.filter.MockUpperCaseFilter;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.pipes.emitter.EmitKey;
 import org.apache.tika.pipes.fetcher.FetchKey;
 
-class PipesClientTest {
+public class PipesClientTest {
     String fetcherName = "fs";
     String testPdfFile = "testOverlappingText.pdf";
 
@@ -49,7 +53,7 @@ class PipesClientTest {
     }
 
     @Test
-    void process() throws IOException, InterruptedException {
+    public void testBasic() throws IOException, InterruptedException {
         PipesResult pipesResult = pipesClient.process(
                 new FetchEmitTuple(testPdfFile, new FetchKey(fetcherName, testPdfFile),
                         new EmitKey(), new Metadata(), new ParseContext(), FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
@@ -58,4 +62,18 @@ class PipesClientTest {
         Metadata metadata = pipesResult.getEmitData().getMetadataList().get(0);
         Assertions.assertEquals("testOverlappingText.pdf", metadata.get("resourceName"));
     }
+
+    @Test
+    public void testMetadataFilter() throws IOException, InterruptedException {
+        ParseContext parseContext = new ParseContext();
+        MetadataFilter metadataFilter = new CompositeMetadataFilter(List.of(new MockUpperCaseFilter()));
+        parseContext.set(MetadataFilter.class, metadataFilter);
+        PipesResult pipesResult = pipesClient.process(
+                new FetchEmitTuple(testPdfFile, new FetchKey(fetcherName, testPdfFile),
+                        new EmitKey(), new Metadata(), parseContext, FetchEmitTuple.ON_PARSE_EXCEPTION.SKIP));
+        Assertions.assertNotNull(pipesResult.getEmitData().getMetadataList());
+        Assertions.assertEquals(1, pipesResult.getEmitData().getMetadataList().size());
+        Metadata metadata = pipesResult.getEmitData().getMetadataList().get(0);
+        Assertions.assertEquals("TESTOVERLAPPINGTEXT.PDF", metadata.get("resourceName"));
+    }
 }