You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tika.apache.org by ta...@apache.org on 2024/08/06 13:28:17 UTC
(tika) 01/01: TIKA-4294 -- further simplifications
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch TIKA-4294c
in repository https://gitbox.apache.org/repos/asf/tika.git
commit 8b353f2cad3a053bb33eda1e98da7430c2161287
Author: tallison <ta...@apache.org>
AuthorDate: Tue Aug 6 09:27:44 2024 -0400
TIKA-4294 -- further simplifications
---
.../serialization/ParseContextDeserializer.java | 2 +-
.../tika/serialization/ParseContextSerializer.java | 2 +-
.../tika/serialization/TikaJsonDeserializer.java | 9 ++--
.../tika/serialization/TikaJsonSerializer.java | 24 +++-------
.../apache/tika/server/standard/TikaPipesTest.java | 54 +++++++++++++++++-----
5 files changed, 54 insertions(+), 37 deletions(-)
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java
index 3f686e4b8..41b7a7834 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextDeserializer.java
@@ -58,7 +58,7 @@ public class ParseContextDeserializer extends JsonDeserializer<ParseContext> {
try {
Class clazz = Class.forName(className);
Class superClazz = className.equals(superClassName) ? clazz : Class.forName(superClassName);
- parseContext.set(superClazz, TikaJsonDeserializer.deserialize(clazz, superClazz, obj));
+ parseContext.set(superClazz, TikaJsonDeserializer.deserialize(clazz, obj));
} catch (ReflectiveOperationException ex) {
throw new IOExceptionWithCause(ex);
}
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java
index 95d6b3698..584920e5c 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/ParseContextSerializer.java
@@ -35,7 +35,7 @@ public class ParseContextSerializer extends JsonSerializer<ParseContext> {
for (String className : parseContext.keySet()) {
try {
Class clazz = Class.forName(className);
- TikaJsonSerializer.serialize(className, parseContext.get(clazz), null, jsonGenerator);
+ TikaJsonSerializer.serialize(className, parseContext.get(clazz), jsonGenerator);
} catch (TikaSerializationException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonDeserializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonDeserializer.java
index 0897b1d91..b346e8775 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonDeserializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonDeserializer.java
@@ -47,18 +47,15 @@ public class TikaJsonDeserializer {
String className = root
.get(TikaJsonSerializer.INSTANTIATED_CLASS_KEY)
.asText();
- String superClass = root.has(TikaJsonSerializer.SUPER_CLASS_KEY) ? root
- .get(TikaJsonSerializer.SUPER_CLASS_KEY)
- .asText() : className;
try {
- return Optional.of(deserialize(Class.forName(className), Class.forName(superClass), root));
+ return Optional.of(deserialize(Class.forName(className), root));
} catch (Exception e) {
throw new IllegalArgumentException(e);
}
}
- public static <T> T deserialize(Class<? extends T> clazz, Class<? extends T> superClazz, JsonNode root) throws ReflectiveOperationException {
+ public static <T> T deserialize(Class<? extends T> clazz, JsonNode root) throws ReflectiveOperationException {
T obj = clazz
.getDeclaredConstructor()
.newInstance();
@@ -71,7 +68,7 @@ public class TikaJsonDeserializer {
Map.Entry<String, JsonNode> e = fields.next();
String name = e.getKey();
JsonNode child = e.getValue();
- if (TikaJsonSerializer.INSTANTIATED_CLASS_KEY.equals(name) || TikaJsonSerializer.SUPER_CLASS_KEY.equals(name)) {
+ if (TikaJsonSerializer.INSTANTIATED_CLASS_KEY.equals(name)) {
continue;
}
setValue(name, child, obj, setters);
diff --git a/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java b/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java
index 4767595a1..a045bffea 100644
--- a/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java
+++ b/tika-serialization/src/main/java/org/apache/tika/serialization/TikaJsonSerializer.java
@@ -43,7 +43,6 @@ import org.slf4j.LoggerFactory;
public class TikaJsonSerializer {
public static String INSTANTIATED_CLASS_KEY = "_class";
- public static String SUPER_CLASS_KEY = "_super_class";
static Set<Class> PRIMITIVES = Set.of(int.class, double.class, float.class, long.class, short.class, boolean.class, String.class, byte.class, char.class);
static Set<Class> BOXED = Set.of(Integer.class, Double.class, Float.class, Long.class, Short.class, Boolean.class, Byte.class, Character.class);
static String SET = "set";
@@ -52,10 +51,10 @@ public class TikaJsonSerializer {
private static String IS = "is";
public static void serialize(Object obj, JsonGenerator jsonGenerator) throws TikaSerializationException, IOException {
- serialize(null, obj, obj.getClass(), jsonGenerator);
+ serialize(null, obj, jsonGenerator);
}
- public static void serialize(String fieldName, Object obj, Class superClass, JsonGenerator jsonGenerator) throws TikaSerializationException, IOException {
+ public static void serialize(String fieldName, Object obj, JsonGenerator jsonGenerator) throws TikaSerializationException, IOException {
if (obj == null) {
if (fieldName == null) {
jsonGenerator.writeNull();
@@ -75,7 +74,7 @@ public class TikaJsonSerializer {
.isEnum()) {
jsonGenerator.writeStringField(fieldName, ((Enum) obj).name());
} else {
- serializeObject(fieldName, obj, superClass, jsonGenerator);
+ serializeObject(fieldName, obj, jsonGenerator);
}
}
@@ -94,11 +93,10 @@ public class TikaJsonSerializer {
/**
* @param fieldName can be null -- used only for logging and debugging
* @param obj
- * @param superClass
* @param jsonGenerator
* @throws TikaSerializationException
*/
- public static void serializeObject(String fieldName, Object obj, Class superClass, JsonGenerator jsonGenerator) throws TikaSerializationException {
+ public static void serializeObject(String fieldName, Object obj, JsonGenerator jsonGenerator) throws TikaSerializationException {
try {
@@ -116,11 +114,6 @@ public class TikaJsonSerializer {
jsonGenerator.writeStringField(INSTANTIATED_CLASS_KEY, obj
.getClass()
.getName());
- if (superClass != null && !obj
- .getClass()
- .equals(superClass)) {
- jsonGenerator.writeStringField(SUPER_CLASS_KEY, superClass.getName());
- }
Map<String, Method> matches = getGetters(obj
.getClass()
.getMethods());
@@ -130,10 +123,7 @@ public class TikaJsonSerializer {
Object methodVal = e
.getValue()
.invoke(obj);
- Class valSuperClass = e
- .getValue()
- .getReturnType();
- serialize(e.getKey(), methodVal, valSuperClass, jsonGenerator);
+ serialize(e.getKey(), methodVal, jsonGenerator);
} catch (IllegalAccessException | InvocationTargetException ex) {
throw new TikaSerializationException("couldn't write paramName=" + e.getKey(), ex);
}
@@ -213,9 +203,7 @@ public class TikaJsonSerializer {
} else if (Map.class.isAssignableFrom(clazz)) {
jsonGenerator.writeStartObject();
for (Map.Entry<String, Object> e : ((Map<String, Object>) obj).entrySet()) {
- serialize(e.getKey(), e.getValue(), e
- .getValue()
- .getClass(), jsonGenerator);
+ serialize(e.getKey(), e.getValue(), jsonGenerator);
}
jsonGenerator.writeEndObject();
} else {
diff --git a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java
index d84c18d41..36dc60a3c 100644
--- a/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java
+++ b/tika-server/tika-server-standard/src/test/java/org/apache/tika/server/standard/TikaPipesTest.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import jakarta.ws.rs.core.Response;
+import org.apache.commons.io.FileUtils;
import org.apache.cxf.jaxrs.JAXRSServerFactoryBean;
import org.apache.cxf.jaxrs.client.WebClient;
import org.apache.cxf.jaxrs.lifecycle.ResourceProvider;
@@ -51,6 +52,7 @@ import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.pdf.PDFParserConfig;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.HandlerConfig;
import org.apache.tika.pipes.emitter.EmitKey;
@@ -76,10 +78,11 @@ public class TikaPipesTest extends CXFTestBase {
private static final String PIPES_PATH = "/pipes";
private static final String TEST_RECURSIVE_DOC = "test_recursive_embedded.docx";
+ private static final String TEST_TWO_BOXES_PDF = "testPDFTwoTextBoxes.pdf";
+
@TempDir
private static Path TMP_WORKING_DIR;
private static Path TMP_OUTPUT_DIR;
- private static Path TMP_OUTPUT_FILE;
private static Path TMP_BYTES_DIR;
private static Path TIKA_PIPES_LOG4j2_PATH;
private static Path TIKA_CONFIG_PATH;
@@ -91,13 +94,13 @@ public class TikaPipesTest extends CXFTestBase {
Path inputDir = TMP_WORKING_DIR.resolve("input");
TMP_OUTPUT_DIR = TMP_WORKING_DIR.resolve("output");
TMP_BYTES_DIR = TMP_WORKING_DIR.resolve("bytes");
- TMP_OUTPUT_FILE = TMP_OUTPUT_DIR.resolve(TEST_RECURSIVE_DOC + ".json");
Files.createDirectories(inputDir);
Files.createDirectories(TMP_OUTPUT_DIR);
Files.copy(TikaPipesTest.class.getResourceAsStream("/test-documents/" + TEST_RECURSIVE_DOC), inputDir.resolve("test_recursive_embedded.docx"),
StandardCopyOption.REPLACE_EXISTING);
-
+ Files.copy(TikaPipesTest.class.getResourceAsStream("/test-documents/" + TEST_TWO_BOXES_PDF), inputDir.resolve(TEST_TWO_BOXES_PDF),
+ StandardCopyOption.REPLACE_EXISTING);
TIKA_CONFIG_PATH = Files.createTempFile(TMP_WORKING_DIR, "tika-pipes-", ".xml");
TIKA_PIPES_LOG4j2_PATH = Files.createTempFile(TMP_WORKING_DIR, "log4j2-", ".xml");
Files.copy(TikaPipesTest.class.getResourceAsStream("/log4j2.xml"), TIKA_PIPES_LOG4j2_PATH, StandardCopyOption.REPLACE_EXISTING);
@@ -120,11 +123,8 @@ public class TikaPipesTest extends CXFTestBase {
@BeforeEach
public void setUpEachTest() throws Exception {
- if (Files.exists(TMP_OUTPUT_FILE)) {
- Files.delete(TMP_OUTPUT_FILE);
- }
-
- assertFalse(Files.isRegularFile(TMP_OUTPUT_FILE));
+ FileUtils.deleteDirectory(TMP_OUTPUT_DIR.toFile());
+ assertFalse(Files.isDirectory(TMP_OUTPUT_DIR));
}
@Override
@@ -173,7 +173,7 @@ public class TikaPipesTest extends CXFTestBase {
assertEquals(200, response.getStatus());
List<Metadata> metadataList = null;
- try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_FILE)) {
+ try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_DIR.resolve(TEST_RECURSIVE_DOC + ".json"))) {
metadataList = JsonMetadataList.fromJson(reader);
}
assertEquals(12, metadataList.size());
@@ -205,7 +205,7 @@ public class TikaPipesTest extends CXFTestBase {
assertEquals(200, response.getStatus());
List<Metadata> metadataList = null;
- try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_FILE)) {
+ try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_DIR.resolve(TEST_RECURSIVE_DOC + ".json"))) {
metadataList = JsonMetadataList.fromJson(reader);
}
assertEquals(1, metadataList.size());
@@ -214,6 +214,38 @@ public class TikaPipesTest extends CXFTestBase {
.get(TikaCoreProperties.TIKA_CONTENT));
}
+ @Test
+ public void testPDFConfig() throws Exception {
+ Metadata metadata = new Metadata();
+ ParseContext parseContext = new ParseContext();
+ PDFParserConfig pdfParserConfig = new PDFParserConfig();
+ pdfParserConfig.setSortByPosition(true);
+ parseContext.set(PDFParserConfig.class, pdfParserConfig);
+
+ FetchEmitTuple t = new FetchEmitTuple("myId", new FetchKey("fsf", TEST_TWO_BOXES_PDF),
+ new EmitKey("fse", ""), metadata, parseContext);
+ StringWriter writer = new StringWriter();
+ JsonFetchEmitTuple.toJson(t, writer);
+ String getUrl = endPoint + PIPES_PATH;
+ Response response = WebClient
+ .create(getUrl)
+ .accept("application/json")
+ .post(writer.toString());
+ assertEquals(200, response.getStatus());
+
+ List<Metadata> metadataList = null;
+ Path outputFile = TMP_OUTPUT_DIR.resolve(TEST_TWO_BOXES_PDF + ".json");
+ try (Reader reader = Files.newBufferedReader(outputFile)) {
+ metadataList = JsonMetadataList.fromJson(reader);
+ }
+ String content = metadataList.get(0).get(TikaCoreProperties.TIKA_CONTENT);
+ content = content.replaceAll("\\s+", " ");
+ // Column text is now interleaved:
+ assertContains(
+ "Left column line 1 Right column line 1 Left colu mn line 2 Right column line 2",
+ content);
+ }
+
@Test
public void testBytes() throws Exception {
EmbeddedDocumentBytesConfig config = new EmbeddedDocumentBytesConfig(true);
@@ -241,7 +273,7 @@ public class TikaPipesTest extends CXFTestBase {
assertEquals(200, response.getStatus());
List<Metadata> metadataList = null;
- try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_FILE)) {
+ try (Reader reader = Files.newBufferedReader(TMP_OUTPUT_DIR.resolve(TEST_RECURSIVE_DOC + ".json"))) {
metadataList = JsonMetadataList.fromJson(reader);
}
assertEquals(12, metadataList.size());