You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jk...@apache.org on 2017/10/26 20:21:30 UTC
[1/3] beam git commit: Many improvements to TikaIO
Repository: beam
Updated Branches:
refs/heads/master e01c78da7 -> 0c2211375
Many improvements to TikaIO
This addresses most of the comments in #3378.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ba93dd39
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ba93dd39
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ba93dd39
Branch: refs/heads/master
Commit: ba93dd39111ab2b13f811d0abeb76a49a4a4f035
Parents: e01c78d
Author: Sergey Beryozkin <sb...@gmail.com>
Authored: Mon Sep 11 16:11:10 2017 +0100
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Oct 26 12:45:19 2017 -0700
----------------------------------------------------------------------
sdks/java/io/tika/pom.xml | 10 -
.../apache/beam/sdk/io/tika/ParseResult.java | 98 ++++
.../org/apache/beam/sdk/io/tika/TikaIO.java | 334 ++++++-------
.../apache/beam/sdk/io/tika/TikaOptions.java | 78 ----
.../org/apache/beam/sdk/io/tika/TikaSource.java | 466 -------------------
.../beam/sdk/io/tika/ParseResultTest.java | 43 ++
.../org/apache/beam/sdk/io/tika/TikaIOTest.java | 252 ++++------
.../apache/beam/sdk/io/tika/TikaReaderTest.java | 82 ----
.../apache/beam/sdk/io/tika/TikaSourceTest.java | 73 ---
9 files changed, 392 insertions(+), 1044 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/pom.xml b/sdks/java/io/tika/pom.xml
index b8f7ece..d7f7e42 100644
--- a/sdks/java/io/tika/pom.xml
+++ b/sdks/java/io/tika/pom.xml
@@ -54,16 +54,6 @@
</dependency>
<dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>${tika.version}</version>
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
new file mode 100644
index 0000000..0a77491
--- /dev/null
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * Tika parse result containing the file location, metadata
+ * and content converted to String.
+ */
+@SuppressWarnings("serial")
+public class ParseResult implements Serializable {
+ private final String fileLocation;
+ private final String content;
+ private final Metadata metadata;
+ private final String[] metadataNames;
+
+ public ParseResult(String fileLocation, String content) {
+ this(fileLocation, content, new Metadata());
+ }
+
+ public ParseResult(String fileLocation, String content, Metadata metadata) {
+ this.fileLocation = fileLocation;
+ this.content = content;
+ this.metadata = metadata;
+ this.metadataNames = metadata.names();
+ }
+
+ /**
+ * Gets a file content.
+ */
+ public String getContent() {
+ return content;
+ }
+
+ /**
+ * Gets a file metadata.
+ */
+ public Metadata getMetadata() {
+ return metadata;
+ }
+
+ /**
+ * Gets a file location.
+ */
+ public String getFileLocation() {
+ return fileLocation;
+ }
+
+ @Override
+ public int hashCode() {
+ int hashCode = 1;
+ hashCode = 31 * hashCode + fileLocation.hashCode();
+ hashCode = 31 * hashCode + content.hashCode();
+ hashCode = 31 * hashCode + getMetadataHashCode();
+ return hashCode;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ParseResult)) {
+ return false;
+ }
+
+ ParseResult pr = (ParseResult) obj;
+ return this.fileLocation.equals(pr.fileLocation)
+ && this.content.equals(pr.content)
+ && this.metadata.equals(pr.metadata);
+ }
+
+ //TODO:
+ // Remove this function and use metadata.hashCode() once Apache Tika 1.17 gets released.
+ private int getMetadataHashCode() {
+ int hashCode = 0;
+ for (String name : metadataNames) {
+ hashCode += name.hashCode() ^ Arrays.hashCode(metadata.getValues(name));
+ }
+ return hashCode;
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
index 4876dcf..32353e1 100644
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
@@ -18,40 +18,72 @@
package org.apache.beam.sdk.io.tika;
import static com.google.common.base.Preconditions.checkNotNull;
+
import com.google.auto.value.AutoValue;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+
import javax.annotation.Nullable;
+
import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.Read.Bounded;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResourceId;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.ToTextContentHandler;
+import org.xml.sax.ContentHandler;
+
+
/**
- * {@link PTransform} for parsing arbitrary files using Apache Tika.
+ * A collection of {@link PTransform} transforms for parsing arbitrary files using Apache Tika.
* Files in many well known text, binary or scientific formats can be processed.
*
- * <p>To read a {@link PCollection} from one or more files
- * use {@link TikaIO.Read#from(String)}
- * to specify the path of the file(s) to be read.
+ * <p>{@link TikaIO.Parse} and {@link TikaIO.ParseAll} parse the files and return
+ * a {@link PCollection} containing one {@link ParseResult} per each file.
+ *
+ * <p>Combine {@link TikaIO.ParseAll} with {@link FileIO.Match}
+ * and {@link FileIO.ReadMatches} to match, read and parse the files.
+ *
+ * <p>Example:
*
- * <p>{@link TikaIO.Read} returns a bounded {@link PCollection} of {@link String Strings},
- * each corresponding to a sequence of characters reported by Apache Tika SAX Parser.
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * // A simple parse of a local PDF file (only runs locally):
+ * PCollection<ParseResult> results =
+ * p.apply(FileIO.match().filepattern("/local/path/to/file.pdf"))
+ * .apply(FileIO.readMatches())
+ * .apply(TikaIO.parseFiles());
+ * }</pre>
+ *
+ * <p>Use {@link TikaIO.Parse} to match, read and parse the files in simple cases.
*
* <p>Example:
*
* <pre>{@code
* Pipeline p = ...;
*
- * // A simple Read of a local PDF file (only runs locally):
- * PCollection<String> content = p.apply(TikaInput.from("/local/path/to/file.pdf"));
+ * // A simple parse of a local PDF file (only runs locally):
+ * PCollection<ParseResult> results =
+ * p.apply(TikaIO.parseAll().filepattern("/local/path/to/file.pdf"));
* }</pre>
*
* <b>Warning:</b> the API of this IO is likely to change in the next release.
@@ -60,85 +92,96 @@ import org.apache.tika.metadata.Metadata;
public class TikaIO {
/**
- * A {@link PTransform} that parses one or more files and returns a bounded {@link PCollection}
- * containing one element for each sequence of characters reported by Apache Tika SAX Parser.
+ * A {@link PTransform} that matches and parses the files
+ * and returns a bounded {@link PCollection} of {@link ParseResult}.
*/
- public static Read read() {
- return new AutoValue_TikaIO_Read.Builder()
- .setQueuePollTime(Read.DEFAULT_QUEUE_POLL_TIME)
- .setQueueMaxPollTime(Read.DEFAULT_QUEUE_MAX_POLL_TIME)
+ public static Parse parse() {
+ return new AutoValue_TikaIO_Parse.Builder()
.build();
- }
+ }
- /** Implementation of {@link #read}. */
- @AutoValue
- public abstract static class Read extends PTransform<PBegin, PCollection<String>> {
- private static final long serialVersionUID = 2198301984784351829L;
- public static final long DEFAULT_QUEUE_POLL_TIME = 50L;
- public static final long DEFAULT_QUEUE_MAX_POLL_TIME = 3000L;
+ /**
+ * A {@link PTransform} that accepts a {@link PCollection} of {@link ReadableFile}
+ * and returns a {@link PCollection} of {@link ParseResult}.
+ */
+ public static ParseAll parseAll() {
+ return new AutoValue_TikaIO_ParseAll.Builder()
+ .build();
+ }
- @Nullable abstract ValueProvider<String> getFilepattern();
- @Nullable abstract ValueProvider<String> getTikaConfigPath();
- @Nullable abstract Metadata getInputMetadata();
- @Nullable abstract Boolean getReadOutputMetadata();
- @Nullable abstract Long getQueuePollTime();
- @Nullable abstract Long getQueueMaxPollTime();
- @Nullable abstract Integer getMinimumTextLength();
- @Nullable abstract Boolean getParseSynchronously();
+ /** Implementation of {@link #parse}. */
+ @SuppressWarnings("serial")
+ @AutoValue
+ public abstract static class Parse extends PTransform<PBegin, PCollection<ParseResult>> {
+ @Nullable
+ abstract ValueProvider<String> getFilepattern();
abstract Builder toBuilder();
@AutoValue.Builder
abstract static class Builder {
abstract Builder setFilepattern(ValueProvider<String> filepattern);
- abstract Builder setTikaConfigPath(ValueProvider<String> tikaConfigPath);
- abstract Builder setInputMetadata(Metadata metadata);
- abstract Builder setReadOutputMetadata(Boolean value);
- abstract Builder setQueuePollTime(Long value);
- abstract Builder setQueueMaxPollTime(Long value);
- abstract Builder setMinimumTextLength(Integer value);
- abstract Builder setParseSynchronously(Boolean value);
- abstract Read build();
+ abstract Parse build();
}
- /**
- * A {@link PTransform} that parses one or more files with the given filename
- * or filename pattern and returns a bounded {@link PCollection} containing
- * one element for each sequence of characters reported by Apache Tika SAX Parser.
- *
- * <p>Filepattern can be a local path (if running locally), or a Google Cloud Storage
- * filename or filename pattern of the form {@code "gs://<bucket>/<filepath>"}
- * (if running locally or using remote execution service).
- *
- * <p>Standard <a href="http://docs.oracle.com/javase/tutorial/essential/io/find.html" >Java
- * Filesystem glob patterns</a> ("*", "?", "[..]") are supported.
- */
- public Read from(String filepattern) {
- checkNotNull(filepattern, "Filepattern cannot be empty.");
- return from(StaticValueProvider.of(filepattern));
+ /** Matches the given filepattern. */
+ public Parse filepattern(String filepattern) {
+ return this.filepattern(ValueProvider.StaticValueProvider.of(filepattern));
}
- /** Same as {@code from(filepattern)}, but accepting a {@link ValueProvider}. */
- public Read from(ValueProvider<String> filepattern) {
- checkNotNull(filepattern, "Filepattern cannot be empty.");
- return toBuilder()
- .setFilepattern(filepattern)
- .setQueuePollTime(Read.DEFAULT_QUEUE_POLL_TIME)
- .setQueueMaxPollTime(Read.DEFAULT_QUEUE_MAX_POLL_TIME)
- .build();
+ /** Like {@link #filepattern(String)} but using a {@link ValueProvider}. */
+ public Parse filepattern(ValueProvider<String> filepattern) {
+ return toBuilder().setFilepattern(filepattern).build();
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder
+ .addIfNotNull(
+ DisplayData.item("filePattern", getFilepattern()).withLabel("File Pattern"));
+ }
+
+ @Override
+ public PCollection<ParseResult> expand(PBegin input) {
+ return input
+ .apply(FileIO.match().filepattern(getFilepattern()))
+ .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+ .apply(parseAll());
+ }
+ }
+
+ /** Implementation of {@link #parseAll}. */
+ @SuppressWarnings("serial")
+ @AutoValue
+ public abstract static class ParseAll extends
+ PTransform<PCollection<ReadableFile>, PCollection<ParseResult>> {
+
+ @Nullable abstract ValueProvider<String> getTikaConfigPath();
+ @Nullable abstract Metadata getInputMetadata();
+
+ abstract Builder toBuilder();
+
+ @AutoValue.Builder
+ abstract static class Builder {
+ abstract Builder setTikaConfigPath(ValueProvider<String> tikaConfigPath);
+ abstract Builder setInputMetadata(Metadata metadata);
+
+ abstract ParseAll build();
}
/**
* Returns a new transform which will use the custom TikaConfig.
*/
- public Read withTikaConfigPath(String tikaConfigPath) {
+ public ParseAll withTikaConfigPath(String tikaConfigPath) {
checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
return withTikaConfigPath(StaticValueProvider.of(tikaConfigPath));
}
/** Same as {@code with(tikaConfigPath)}, but accepting a {@link ValueProvider}. */
- public Read withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
+ public ParseAll withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
return toBuilder()
.setTikaConfigPath(tikaConfigPath)
@@ -149,7 +192,7 @@ public class TikaIO {
* Returns a new transform which will use the provided content type hint
* to make the file parser detection more efficient.
*/
- public Read withContentTypeHint(String contentType) {
+ public ParseAll withContentTypeHint(String contentType) {
checkNotNull(contentType, "ContentType cannot be empty.");
Metadata metadata = new Metadata();
metadata.add(Metadata.CONTENT_TYPE, contentType);
@@ -160,7 +203,7 @@ public class TikaIO {
* Returns a new transform which will use the provided input metadata
* for parsing the files.
*/
- public Read withInputMetadata(Metadata metadata) {
+ public ParseAll withInputMetadata(Metadata metadata) {
Metadata inputMetadata = this.getInputMetadata();
if (inputMetadata != null) {
for (String name : metadata.names()) {
@@ -172,88 +215,15 @@ public class TikaIO {
return toBuilder().setInputMetadata(inputMetadata).build();
}
- /**
- * Returns a new transform which will report the metadata.
- */
- public Read withReadOutputMetadata(Boolean value) {
- return toBuilder().setReadOutputMetadata(value).build();
- }
-
- /**
- * Returns a new transform which will use the specified queue poll time.
- */
- public Read withQueuePollTime(Long value) {
- return toBuilder().setQueuePollTime(value).build();
- }
-
- /**
- * Returns a new transform which will use the specified queue max poll time.
- */
- public Read withQueueMaxPollTime(Long value) {
- return toBuilder().setQueueMaxPollTime(value).build();
- }
-
- /**
- * Returns a new transform which will operate on the text blocks with the
- * given minimum text length.
- */
- public Read withMinimumTextlength(Integer value) {
- return toBuilder().setMinimumTextLength(value).build();
- }
-
- /**
- * Returns a new transform which will use the synchronous reader.
- */
- public Read withParseSynchronously(Boolean value) {
- return toBuilder().setParseSynchronously(value).build();
- }
-
- /**
- * Path to Tika configuration resource.
- */
- public Read withOptions(TikaOptions options) {
- checkNotNull(options, "TikaOptions cannot be empty.");
- Builder builder = toBuilder();
- builder.setFilepattern(StaticValueProvider.of(options.getInput()))
- .setQueuePollTime(options.getQueuePollTime())
- .setQueueMaxPollTime(options.getQueueMaxPollTime())
- .setMinimumTextLength(options.getMinimumTextLength())
- .setParseSynchronously(options.getParseSynchronously());
- if (options.getContentTypeHint() != null) {
- Metadata metadata = this.getInputMetadata();
- if (metadata == null) {
- metadata = new Metadata();
- }
- metadata.add(Metadata.CONTENT_TYPE, options.getContentTypeHint());
- builder.setInputMetadata(metadata);
- }
- if (options.getTikaConfigPath() != null) {
- builder.setTikaConfigPath(StaticValueProvider.of(options.getTikaConfigPath()));
- }
- if (Boolean.TRUE.equals(options.getReadOutputMetadata())) {
- builder.setReadOutputMetadata(options.getReadOutputMetadata());
- }
- return builder.build();
- }
-
@Override
- public PCollection<String> expand(PBegin input) {
- checkNotNull(this.getFilepattern(), "Filepattern cannot be empty.");
- final Bounded<String> read = org.apache.beam.sdk.io.Read.from(new TikaSource(this));
- PCollection<String> pcol = input.getPipeline().apply(read);
- pcol.setCoder(getDefaultOutputCoder());
- return pcol;
+ public PCollection<ParseResult> expand(PCollection<ReadableFile> input) {
+ return input.apply(ParDo.of(new ParseToStringFn(this)));
}
@Override
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- String filepatternDisplay = getFilepattern().isAccessible()
- ? getFilepattern().get() : getFilepattern().toString();
- builder
- .addIfNotNull(DisplayData.item("filePattern", filepatternDisplay)
- .withLabel("File Pattern"));
if (getTikaConfigPath() != null) {
String tikaConfigPathDisplay = getTikaConfigPath().isAccessible()
? getTikaConfigPath().get() : getTikaConfigPath().toString();
@@ -262,49 +232,55 @@ public class TikaIO {
}
Metadata metadata = getInputMetadata();
if (metadata != null) {
- StringBuilder sb = new StringBuilder();
- sb.append('[');
- for (String name : metadata.names()) {
- if (sb.length() > 1) {
- sb.append(',');
- }
- sb.append(name).append('=').append(metadata.get(name));
- }
- sb.append(']');
+ //TODO: use metadata.toString() only without a trim() once Apache Tika 1.17 gets released
builder
- .add(DisplayData.item("inputMetadata", sb.toString())
+ .add(DisplayData.item("inputMetadata", metadata.toString().trim())
.withLabel("Input Metadata"));
}
- if (Boolean.TRUE.equals(getParseSynchronously())) {
- builder
- .add(DisplayData.item("parseMode", "synchronous")
- .withLabel("Parse Mode"));
- } else {
- builder
- .add(DisplayData.item("parseMode", "asynchronous")
- .withLabel("Parse Mode"));
- builder
- .add(DisplayData.item("queuePollTime", getQueuePollTime().toString())
- .withLabel("Queue Poll Time"))
- .add(DisplayData.item("queueMaxPollTime", getQueueMaxPollTime().toString())
- .withLabel("Queue Max Poll Time"));
- }
- Integer minTextLen = getMinimumTextLength();
- if (minTextLen != null && minTextLen > 0) {
- builder
- .add(DisplayData.item("minTextLen", getMinimumTextLength().toString())
- .withLabel("Minimum Text Length"));
+ }
+
+ private static class ParseToStringFn extends DoFn<ReadableFile, ParseResult> {
+
+ private static final long serialVersionUID = 6837207505313720989L;
+ private final TikaIO.ParseAll spec;
+ private TikaConfig tikaConfig;
+
+ ParseToStringFn(TikaIO.ParseAll spec) {
+ this.spec = spec;
}
- if (Boolean.TRUE.equals(getReadOutputMetadata())) {
- builder
- .add(DisplayData.item("readOutputMetadata", "true")
- .withLabel("Read Output Metadata"));
+
+ @Setup
+ public void setup() throws Exception {
+ if (spec.getTikaConfigPath() != null) {
+ ResourceId configResource =
+ FileSystems.matchSingleFileSpec(spec.getTikaConfigPath().get()).resourceId();
+ tikaConfig = new TikaConfig(
+ Channels.newInputStream(FileSystems.open(configResource)));
+ }
}
- }
- @Override
- protected Coder<String> getDefaultOutputCoder() {
- return StringUtf8Coder.of();
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ ReadableFile file = c.element();
+ InputStream stream = Channels.newInputStream(file.open());
+ try (InputStream tikaStream = TikaInputStream.get(stream)) {
+
+ final Parser parser = tikaConfig == null
+ ? new AutoDetectParser() : new AutoDetectParser(tikaConfig);
+
+ final ParseContext context = new ParseContext();
+ context.set(Parser.class, parser);
+ Metadata tikaMetadata = spec.getInputMetadata() != null
+ ? spec.getInputMetadata() : new org.apache.tika.metadata.Metadata();
+
+ ContentHandler tikaHandler = new ToTextContentHandler();
+ parser.parse(tikaStream, tikaHandler, tikaMetadata, context);
+
+ c.output(new ParseResult(file.getMetadata().resourceId().toString(),
+ tikaHandler.toString(),
+ tikaMetadata));
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java
deleted file mode 100644
index fb97678..0000000
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import org.apache.beam.sdk.options.Default;
-import org.apache.beam.sdk.options.Description;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.Validation;
-
-/**
- * TikaInput Options to support the command-line applications.
- */
-public interface TikaOptions extends PipelineOptions {
-
- @Description("Input path")
- @Validation.Required
- String getInput();
- void setInput(String value);
-
- @Description("Tika Config path")
- String getTikaConfigPath();
- void setTikaConfigPath(String value);
-
- @Description("Tika Parser Content Type hint")
- String getContentTypeHint();
- void setContentTypeHint(String value);
-
- @Description("Metadata report status")
- @Default.Boolean(false)
- Boolean getReadOutputMetadata();
- void setReadOutputMetadata(Boolean value);
-
- @Description("Optional use of the synchronous reader")
- @Default.Boolean(false)
- Boolean getParseSynchronously();
- void setParseSynchronously(Boolean value);
-
- @Description("Tika Parser queue poll time in milliseconds")
- @Default.Long(TikaIO.Read.DEFAULT_QUEUE_POLL_TIME)
- Long getQueuePollTime();
- void setQueuePollTime(Long value);
-
- @Description("Tika Parser queue maximum poll time in milliseconds")
- @Default.Long(TikaIO.Read.DEFAULT_QUEUE_MAX_POLL_TIME)
- Long getQueueMaxPollTime();
- void setQueueMaxPollTime(Long value);
-
- @Description("Minumin text fragment length for Tika Parser to report")
- @Default.Integer(0)
- Integer getMinimumTextLength();
- void setMinimumTextLength(Integer value);
-
- @Description("Pipeline name")
- @Default.String("TikaRead")
- String getPipelineName();
- void setPipelineName(String value);
-
- @Description("Output path")
- @Default.String("/tmp/tika/out")
- String getOutput();
- void setOutput(String value);
-
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java
deleted file mode 100644
index 7c8852b..0000000
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.NoSuchElementException;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import javax.annotation.Nullable;
-
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.FileSystems;
-import org.apache.beam.sdk.io.Source;
-import org.apache.beam.sdk.io.fs.MatchResult;
-import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.joda.time.Instant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-/**
- * Implementation detail of {@link TikaIO.Read}.
- *
- * <p>A {@link Source} which can represent the content of the files parsed by Apache Tika.
- */
-class TikaSource extends BoundedSource<String> {
- private static final long serialVersionUID = -509574062910491122L;
- private static final Logger LOG = LoggerFactory.getLogger(TikaSource.class);
-
- @Nullable
- private MatchResult.Metadata singleFileMetadata;
- private final Mode mode;
- private final TikaIO.Read spec;
-
- /**
- * Source mode.
- */
- public enum Mode {
- FILEPATTERN, SINGLE_FILE
- }
-
- TikaSource(TikaIO.Read spec) {
- this.mode = Mode.FILEPATTERN;
- this.spec = spec;
- }
-
- TikaSource(Metadata fileMetadata, TikaIO.Read spec) {
- mode = Mode.SINGLE_FILE;
- this.singleFileMetadata = checkNotNull(fileMetadata, "fileMetadata");
- this.spec = spec;
- }
-
- @Override
- public BoundedReader<String> createReader(PipelineOptions options) throws IOException {
- this.validate();
- checkState(spec.getFilepattern().isAccessible(),
- "Cannot create a Tika reader without access to the file"
- + " or pattern specification: {}.", spec.getFilepattern());
- if (spec.getTikaConfigPath() != null) {
- checkState(spec.getTikaConfigPath().isAccessible(),
- "Cannot create a Tika reader without access to its configuration",
- spec.getTikaConfigPath());
- }
-
- String fileOrPattern = spec.getFilepattern().get();
- if (mode == Mode.FILEPATTERN) {
- List<Metadata> fileMetadata = expandFilePattern(fileOrPattern);
- List<TikaReader> fileReaders = new ArrayList<>();
- for (Metadata metadata : fileMetadata) {
- fileReaders.add(new TikaReader(this, metadata.resourceId().toString()));
- }
- if (fileReaders.size() == 1) {
- return fileReaders.get(0);
- }
- return new FilePatternTikaReader(this, fileReaders);
- } else {
- return new TikaReader(this, singleFileMetadata.resourceId().toString());
- }
-
- }
-
- @Override
- public List<? extends TikaSource> split(long desiredBundleSizeBytes, PipelineOptions options)
- throws Exception {
- if (mode == Mode.SINGLE_FILE) {
- return ImmutableList.of(this);
- } else {
- List<Metadata> fileMetadata = expandFilePattern(spec.getFilepattern().get());
-
- List<TikaSource> splitResults = new LinkedList<>();
- for (Metadata metadata : fileMetadata) {
- splitResults.add(new TikaSource(metadata, spec));
- }
- return splitResults;
- }
- }
-
- public TikaIO.Read getTikaInputRead() {
- return spec;
- }
-
- @Override
- public Coder<String> getDefaultOutputCoder() {
- return StringUtf8Coder.of();
- }
-
- @Override
- public void validate() {
- switch (mode) {
- case FILEPATTERN:
- checkArgument(this.singleFileMetadata == null,
- "Unexpected initialized singleFileMetadata value");
- break;
- case SINGLE_FILE:
- checkNotNull(this.singleFileMetadata,
- "Unexpected null singleFileMetadata value");
- break;
- default:
- throw new IllegalStateException("Unknown mode: " + mode);
- }
- }
-
- @Override
- public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
- long totalSize = 0;
- List<Metadata> fileMetadata = expandFilePattern(spec.getFilepattern().get());
- for (Metadata metadata : fileMetadata) {
- totalSize += metadata.sizeBytes();
- }
- return totalSize;
- }
-
- Mode getMode() {
- return this.mode;
- }
-
- Metadata getSingleFileMetadata() {
- return this.singleFileMetadata;
- }
-
- private static List<Metadata> expandFilePattern(String fileOrPattern) throws IOException {
- MatchResult matches = Iterables.getOnlyElement(
- FileSystems.match(Collections.singletonList(fileOrPattern)));
- LOG.info("Matched {} files for pattern {}", matches.metadata().size(), fileOrPattern);
- List<Metadata> metadata = ImmutableList.copyOf(matches.metadata());
- checkArgument(!metadata.isEmpty(),
- "Unable to find any files matching %s", fileOrPattern);
-
- return metadata;
- }
-
- /**
- * FilePatternTikaReader.
- * TODO: This is mostly a copy of FileBasedSource internal file-pattern reader
- * so that code would need to be generalized as part of the future contribution
- */
- static class FilePatternTikaReader extends BoundedReader<String> {
- private final TikaSource source;
- final ListIterator<TikaReader> fileReadersIterator;
- TikaReader currentReader = null;
-
- public FilePatternTikaReader(TikaSource source, List<TikaReader> fileReaders) {
- this.source = source;
- this.fileReadersIterator = fileReaders.listIterator();
- }
-
- @Override
- public boolean start() throws IOException {
- return startNextNonemptyReader();
- }
-
- @Override
- public boolean advance() throws IOException {
- checkState(currentReader != null, "Call start() before advance()");
- if (currentReader.advance()) {
- return true;
- }
- return startNextNonemptyReader();
- }
-
- private boolean startNextNonemptyReader() throws IOException {
- while (fileReadersIterator.hasNext()) {
- currentReader = fileReadersIterator.next();
- if (currentReader.start()) {
- return true;
- }
- currentReader.close();
- }
- return false;
- }
-
- @Override
- public String getCurrent() throws NoSuchElementException {
- return currentReader.getCurrent();
- }
-
- @Override
- public Instant getCurrentTimestamp() throws NoSuchElementException {
- return currentReader.getCurrentTimestamp();
- }
-
- @Override
- public void close() throws IOException {
- if (currentReader != null) {
- currentReader.close();
- }
- while (fileReadersIterator.hasNext()) {
- fileReadersIterator.next().close();
- }
- }
-
- @Override
- public TikaSource getCurrentSource() {
- return source;
- }
- }
-
- static class TikaReader extends BoundedReader<String> {
- private ExecutorService execService;
- private final ContentHandlerImpl tikaHandler = new ContentHandlerImpl();
- private String current;
- private TikaSource source;
- private String filePath;
- private TikaIO.Read spec;
- private org.apache.tika.metadata.Metadata tikaMetadata;
- private Iterator<String> metadataIterator;
-
- TikaReader(TikaSource source, String filePath) {
- this.source = source;
- this.filePath = filePath;
- this.spec = source.getTikaInputRead();
- }
-
- @Override
- public boolean start() throws IOException {
- final InputStream is = TikaInputStream.get(Paths.get(filePath));
- TikaConfig tikaConfig = null;
- if (spec.getTikaConfigPath() != null) {
- try {
- tikaConfig = new TikaConfig(spec.getTikaConfigPath().get());
- } catch (TikaException | SAXException e) {
- throw new IOException(e);
- }
- }
- final Parser parser = tikaConfig == null ? new AutoDetectParser()
- : new AutoDetectParser(tikaConfig);
- final ParseContext context = new ParseContext();
- context.set(Parser.class, parser);
- tikaMetadata = spec.getInputMetadata() != null ? spec.getInputMetadata()
- : new org.apache.tika.metadata.Metadata();
-
- if (spec.getMinimumTextLength() != null) {
- tikaHandler.setMinTextLength(spec.getMinimumTextLength());
- }
-
- if (!Boolean.TRUE.equals(spec.getParseSynchronously())) {
- // Try to parse the file on the executor thread to make the best effort
- // at letting the pipeline thread advancing over the file content
- // without immediately parsing all of it
- execService = Executors.newFixedThreadPool(1);
- execService.submit(new Runnable() {
- public void run() {
- try {
- parser.parse(is, tikaHandler, tikaMetadata, context);
- is.close();
- } catch (Exception ex) {
- tikaHandler.setParseException(ex);
- }
- }
- });
- } else {
- // Some parsers might not be able to report the content in chunks.
- // It does not make sense to create extra threads in such cases
- try {
- parser.parse(is, tikaHandler, tikaMetadata, context);
- } catch (Exception ex) {
- throw new IOException(ex);
- } finally {
- is.close();
- }
- }
- return advanceToNext();
- }
-
- @Override
- public boolean advance() throws IOException {
- checkState(current != null, "Call start() before advance()");
- return advanceToNext();
- }
-
- protected boolean advanceToNext() throws IOException {
- current = null;
- // The content is reported first
- if (metadataIterator == null) {
- // Check if some content is already available
- current = tikaHandler.getCurrent();
-
- if (current == null && !Boolean.TRUE.equals(spec.getParseSynchronously())) {
- long maxPollTime = 0;
- long configuredMaxPollTime = spec.getQueueMaxPollTime() == null
- ? TikaIO.Read.DEFAULT_QUEUE_MAX_POLL_TIME : spec.getQueueMaxPollTime();
- long configuredPollTime = spec.getQueuePollTime() == null
- ? TikaIO.Read.DEFAULT_QUEUE_POLL_TIME : spec.getQueuePollTime();
-
- // Poll the queue till the next piece of data is available
- while (current == null && maxPollTime < configuredMaxPollTime) {
- boolean docEnded = tikaHandler.waitForNext(configuredPollTime);
- current = tikaHandler.getCurrent();
- // End of Document ?
- if (docEnded) {
- break;
- }
- maxPollTime += spec.getQueuePollTime();
- }
- }
- // No more content ?
- if (current == null && Boolean.TRUE.equals(spec.getReadOutputMetadata())) {
- // Time to report the metadata
- metadataIterator = Arrays.asList(tikaMetadata.names()).iterator();
- }
- }
-
- if (metadataIterator != null && metadataIterator.hasNext()) {
- String key = metadataIterator.next();
- // The metadata name/value separator can be configured if needed
- current = key + "=" + tikaMetadata.get(key);
- }
- return current != null;
- }
-
- @Override
- public String getCurrent() throws NoSuchElementException {
- if (current == null) {
- throw new NoSuchElementException();
- }
- return current;
- }
-
- @Override
- public void close() throws IOException {
- if (execService != null) {
- execService.shutdown();
- }
- }
-
- ExecutorService getExecutorService() {
- return execService;
- }
-
- @Override
- public BoundedSource<String> getCurrentSource() {
- return source;
- }
- }
-
- /**
- * Tika Parser Content Handler.
- */
- static class ContentHandlerImpl extends DefaultHandler {
- private Queue<String> queue = new ConcurrentLinkedQueue<>();
- private volatile boolean documentEnded;
- private volatile Exception parseException;
- private volatile String current;
- private int minTextLength;
-
- @Override
- public void characters(char ch[], int start, int length) throws SAXException {
- String value = new String(ch, start, length).trim();
- if (!value.isEmpty()) {
- if (minTextLength <= 0) {
- queue.add(value);
- } else {
- current = current == null ? value : current + " " + value;
- if (current.length() >= minTextLength) {
- queue.add(current);
- current = null;
- }
- }
- }
- }
-
- public void setParseException(Exception ex) {
- this.parseException = ex;
- }
-
- public synchronized boolean waitForNext(long pollTime) throws IOException {
- if (!documentEnded) {
- try {
- wait(pollTime);
- } catch (InterruptedException ex) {
- // continue;
- }
- }
- return documentEnded;
- }
-
- @Override
- public synchronized void endDocument() throws SAXException {
- this.documentEnded = true;
- notify();
- }
-
- public String getCurrent() throws IOException {
- checkParseException();
- String value = queue.poll();
- if (value == null && documentEnded) {
- return current;
- } else {
- return value;
- }
- }
- public void checkParseException() throws IOException {
- if (parseException != null) {
- throw new IOException(parseException);
- }
- }
-
- public void setMinTextLength(int minTextLength) {
- this.minTextLength = minTextLength;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
new file mode 100644
index 0000000..fd86152
--- /dev/null
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import static org.junit.Assert.assertEquals;
+import org.apache.tika.metadata.Metadata;
+import org.junit.Test;
+
+/**
+ * Tests ParseResult.
+ */
+public class ParseResultTest {
+ @Test
+ public void testEqualsAndHashCode() {
+ ParseResult p1 = new ParseResult("a.txt", "hello", getMetadata());
+ ParseResult p2 = new ParseResult("a.txt", "hello", getMetadata());
+ assertEquals(p1, p2);
+ assertEquals(p1.hashCode(), p2.hashCode());
+ }
+
+ static Metadata getMetadata() {
+ Metadata m = new Metadata();
+ m.add("Author", "BeamTikaUser");
+ m.add("Author", "BeamTikaUser2");
+ m.add("Date", "2017-09-01");
+ return m;
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
index 40ff569..a985b0a 100644
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
@@ -18,14 +18,14 @@
package org.apache.beam.sdk.io.tika;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.hamcrest.Matchers.isA;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import java.io.IOException;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.DoFn;
@@ -33,231 +33,171 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.PCollection;
import org.apache.tika.exception.TikaException;
-import org.junit.Ignore;
+import org.apache.tika.metadata.Metadata;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
/**
* Tests TikaInput.
*/
public class TikaIOTest {
- private static final String[] PDF_FILE = new String[] {
- "Combining", "can help to ingest", "Apache Beam", "in most known formats.",
- "the content from the files", "and", "Apache Tika"
- };
- private static final String[] PDF_ZIP_FILE = new String[] {
- "Combining", "can help to ingest", "Apache Beam", "in most known formats.",
- "the content from the files", "and", "Apache Tika",
- "apache-beam-tika.pdf"
- };
- private static final String[] ODT_FILE = new String[] {
- "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
- "the content from the files", "and", "Apache Tika"
- };
- private static final String[] ODT_FILE_WITH_METADATA = new String[] {
- "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
- "the content from the files", "and", "Apache Tika",
- "Author=BeamTikaUser"
- };
- private static final String[] ODT_FILE_WITH_MIN_TEXT_LEN = new String[] {
- "Combining Apache Beam", "and Apache Tika can help to ingest", "in most known formats.",
- "the content from the files"
- };
- private static final String[] ODT_FILES = new String[] {
- "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
- "the content from the files", "and", "Apache Tika",
- "Open Office", "Text", "PDF", "Excel", "Scientific",
- "and other formats", "are supported."
- };
+ private static final String PDF_FILE =
+ "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+ + "Combining\n\nApache Beam\n\nand\n\nApache Tika\n\ncan help to ingest\n\n"
+ + "the content from the files\n\nin most known formats.\n\n\n";
+
+ private static final String PDF_ZIP_FILE =
+ "\n\n\n\n\n\n\n\napache-beam-tika.pdf\n\n\nCombining\n\n\nApache Beam\n\n\n"
+ + "and\n\n\nApache Tika\n\n\ncan help to ingest\n\n\nthe content from the files\n\n\n"
+ + "in most known formats.\n\n\n\n\n\n\n";
+
+ private static final String ODT_FILE =
+ "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+ + "Combining\nApache Beam\nand\nApache Tika\ncan help to ingest\nthe content from the"
+ + " files\nin most known formats.\n";
+
+ private static final String ODT_FILE2 =
+ "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+ + "Open Office\nPDF\nExcel\nText\nScientific\nand other formats\nare supported.\n";
@Rule
public TestPipeline p = TestPipeline.create();
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
- @Ignore
@Test
- public void testReadPdfFile() throws IOException {
+ public void testParsePdfFile() throws IOException {
String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
- doTestReadFiles(resourcePath, PDF_FILE);
+ doTestParse(resourcePath, new ParseResult(resourcePath, PDF_FILE));
}
- @Test
- public void testReadZipPdfFile() throws IOException {
-
- String resourcePath = getClass().getResource("/apache-beam-tika-pdf.zip").getPath();
-
- doTestReadFiles(resourcePath, PDF_ZIP_FILE);
+ private void doTestParse(String resourcePath, ParseResult... expectedResults)
+ throws IOException {
+ PCollection<ParseResult> output =
+ p.apply("ParseAll", TikaIO.parse().filepattern(resourcePath))
+ .apply(ParDo.of(new FilterMetadataFn()));
+ PAssert.that(output).containsInAnyOrder(expectedResults);
+ p.run();
}
@Test
- public void testReadOdtFile() throws IOException {
+ public void testParseAllPdfFile() throws IOException {
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
+ String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
- doTestReadFiles(resourcePath, ODT_FILE);
+ doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_FILE));
}
@Test
- public void testReadOdtFiles() throws IOException {
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
- resourcePath = resourcePath.replace("apache-beam-tika1", "*");
+ public void testParseAllZipPdfFile() throws IOException {
- doTestReadFiles(resourcePath, ODT_FILES);
- }
+ String resourcePath = getClass().getResource("/apache-beam-tika-pdf.zip").getPath();
- private void doTestReadFiles(String resourcePath, String[] expected) throws IOException {
- PCollection<String> output = p.apply("ParseFiles", TikaIO.read().from(resourcePath));
- PAssert.that(output).containsInAnyOrder(expected);
- p.run();
+ doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_ZIP_FILE));
}
@Test
- public void testReadOdtFileWithMetadata() throws IOException {
+ public void testParseAllOdtFile() throws IOException {
String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
- PCollection<String> output = p.apply("ParseOdtFile",
- TikaIO.read().from(resourcePath).withReadOutputMetadata(true))
- .apply(ParDo.of(new FilterMetadataFn()));
- PAssert.that(output).containsInAnyOrder(ODT_FILE_WITH_METADATA);
- p.run();
+ doTestParseAll(resourcePath, new ParseResult(resourcePath, ODT_FILE, getOdtMetadata()));
}
@Test
- public void testReadOdtFileWithMinTextLength() throws IOException {
+ public void testParseAllOdtFiles() throws IOException {
+ String resourcePath1 = getClass().getResource("/apache-beam-tika1.odt").getPath();
+ String resourcePath2 = getClass().getResource("/apache-beam-tika2.odt").getPath();
+ String resourcePath = resourcePath1.replace("apache-beam-tika1", "*");
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-
- PCollection<String> output = p.apply("ParseOdtFile",
- TikaIO.read().from(resourcePath).withMinimumTextlength(20));
- PAssert.that(output).containsInAnyOrder(ODT_FILE_WITH_MIN_TEXT_LEN);
- p.run();
+ doTestParseAll(resourcePath, new ParseResult(resourcePath1, ODT_FILE, getOdtMetadata()),
+ new ParseResult(resourcePath2, ODT_FILE2));
}
- @Test
- public void testReadPdfFileSync() throws IOException {
-
- String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
-
- PCollection<String> output = p.apply("ParsePdfFile",
- TikaIO.read().from(resourcePath).withParseSynchronously(true));
- PAssert.that(output).containsInAnyOrder(PDF_FILE);
+ private void doTestParseAll(String resourcePath, ParseResult... expectedResults)
+ throws IOException {
+ PCollection<ParseResult> output =
+ p.apply("ParseFiles", FileIO.match().filepattern(resourcePath))
+ .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+ .apply(TikaIO.parseAll())
+ .apply(ParDo.of(new FilterMetadataFn()));
+ PAssert.that(output).containsInAnyOrder(expectedResults);
p.run();
}
@Test
- public void testReadDamagedPdfFile() throws IOException {
+ public void testParseAllDamagedPdfFile() throws IOException {
+ thrown.expectCause(isA(TikaException.class));
+ String resourcePath = getClass().getResource("/damaged.pdf").getPath();
- doTestReadDamagedPdfFile(false);
+ p.apply("ParseInvalidPdfFile", FileIO.match().filepattern(resourcePath))
+ .apply(FileIO.readMatches())
+ .apply(TikaIO.parseAll());
+ p.run();
}
@Test
- public void testReadDamagedPdfFileSync() throws IOException {
- doTestReadDamagedPdfFile(true);
- }
+ public void testParseDisplayData() {
+ TikaIO.Parse parse = TikaIO.parse().filepattern("file.pdf");
- private void doTestReadDamagedPdfFile(boolean sync) throws IOException {
+ DisplayData displayData = DisplayData.from(parse);
- String resourcePath = getClass().getResource("/damaged.pdf").getPath();
-
- p.apply("ParseInvalidPdfFile",
- TikaIO.read().from(resourcePath).withParseSynchronously(sync));
- try {
- p.run();
- fail("Transform failure is expected");
- } catch (RuntimeException ex) {
- assertTrue(ex.getCause().getCause() instanceof TikaException);
- }
+ assertThat(displayData, hasDisplayItem("filePattern", "file.pdf"));
+ assertEquals(1, displayData.items().size());
}
@Test
- public void testReadDisplayData() {
- TikaIO.Read read = TikaIO.read()
- .from("foo.*")
+ public void testParseAllDisplayData() {
+ TikaIO.ParseAll parseAll = TikaIO.parseAll()
.withTikaConfigPath("tikaconfigpath")
- .withContentTypeHint("application/pdf")
- .withMinimumTextlength(100)
- .withReadOutputMetadata(true);
+ .withContentTypeHint("application/pdf");
- DisplayData displayData = DisplayData.from(read);
+ DisplayData displayData = DisplayData.from(parseAll);
- assertThat(displayData, hasDisplayItem("filePattern", "foo.*"));
assertThat(displayData, hasDisplayItem("tikaConfigPath", "tikaconfigpath"));
assertThat(displayData, hasDisplayItem("inputMetadata",
- "[Content-Type=application/pdf]"));
- assertThat(displayData, hasDisplayItem("readOutputMetadata", "true"));
- assertThat(displayData, hasDisplayItem("parseMode", "asynchronous"));
- assertThat(displayData, hasDisplayItem("queuePollTime", "50"));
- assertThat(displayData, hasDisplayItem("queueMaxPollTime", "3000"));
- assertThat(displayData, hasDisplayItem("minTextLen", "100"));
- assertEquals(8, displayData.items().size());
- }
-
- @Test
- public void testReadDisplayDataSyncMode() {
- TikaIO.Read read = TikaIO.read()
- .from("foo.*")
- .withParseSynchronously(true);
-
- DisplayData displayData = DisplayData.from(read);
-
- assertThat(displayData, hasDisplayItem("filePattern", "foo.*"));
- assertThat(displayData, hasDisplayItem("parseMode", "synchronous"));
+ "Content-Type=application/pdf"));
assertEquals(2, displayData.items().size());
}
@Test
- public void testReadDisplayDataWithDefaultOptions() {
- final String[] args = new String[]{"--input=/input/tika.pdf"};
- TikaIO.Read read = TikaIO.read().withOptions(createOptions(args));
+ public void testParseAllDisplayDataWithCustomOptions() {
+ TikaIO.ParseAll parseAll = TikaIO.parseAll()
+ .withTikaConfigPath("/tikaConfigPath")
+ .withContentTypeHint("application/pdf");
- DisplayData displayData = DisplayData.from(read);
+ DisplayData displayData = DisplayData.from(parseAll);
- assertThat(displayData, hasDisplayItem("filePattern", "/input/tika.pdf"));
- assertThat(displayData, hasDisplayItem("parseMode", "asynchronous"));
- assertThat(displayData, hasDisplayItem("queuePollTime", "50"));
- assertThat(displayData, hasDisplayItem("queueMaxPollTime", "3000"));
- assertEquals(4, displayData.items().size());
- }
- @Test
- public void testReadDisplayDataWithCustomOptions() {
- final String[] args = new String[]{"--input=/input/tika.pdf",
- "--tikaConfigPath=/tikaConfigPath",
- "--queuePollTime=10",
- "--queueMaxPollTime=1000",
- "--contentTypeHint=application/pdf",
- "--readOutputMetadata=true"};
- TikaIO.Read read = TikaIO.read().withOptions(createOptions(args));
-
- DisplayData displayData = DisplayData.from(read);
-
- assertThat(displayData, hasDisplayItem("filePattern", "/input/tika.pdf"));
assertThat(displayData, hasDisplayItem("tikaConfigPath", "/tikaConfigPath"));
- assertThat(displayData, hasDisplayItem("parseMode", "asynchronous"));
- assertThat(displayData, hasDisplayItem("queuePollTime", "10"));
- assertThat(displayData, hasDisplayItem("queueMaxPollTime", "1000"));
assertThat(displayData, hasDisplayItem("inputMetadata",
- "[Content-Type=application/pdf]"));
- assertThat(displayData, hasDisplayItem("readOutputMetadata", "true"));
- assertEquals(7, displayData.items().size());
- }
-
- private static TikaOptions createOptions(String[] args) {
- return PipelineOptionsFactory.fromArgs(args)
- .withValidation().as(TikaOptions.class);
+ "Content-Type=application/pdf"));
+ assertEquals(2, displayData.items().size());
}
- static class FilterMetadataFn extends DoFn<String, String> {
+ static class FilterMetadataFn extends DoFn<ParseResult, ParseResult> {
private static final long serialVersionUID = 6338014219600516621L;
@ProcessElement
public void processElement(ProcessContext c) {
- String word = c.element();
- if (word.contains("=") && !word.startsWith("Author")) {
- return;
+ ParseResult result = c.element();
+ Metadata m = new Metadata();
+ // Files contain many metadata properties. This function drops all but the "Author"
+ // property manually added to "apache-beam-tika1.odt" resource only to make
+ // the tests simpler
+ if (result.getFileLocation().endsWith("apache-beam-tika1.odt")) {
+ m.set("Author", result.getMetadata().get("Author"));
}
- c.output(word);
+ ParseResult newResult = new ParseResult(result.getFileLocation(), result.getContent(), m);
+ c.output(newResult);
}
}
+
+ static Metadata getOdtMetadata() {
+ Metadata m = new Metadata();
+ m.set("Author", "BeamTikaUser");
+ return m;
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java
deleted file mode 100644
index 5c4e754..0000000
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.beam.sdk.io.tika.TikaSource.TikaReader;
-import org.junit.Test;
-
-/**
- * Tests TikaReader.
- */
-public class TikaReaderTest {
- private static final List<String> ODT_FILE = Arrays.asList(
- "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
- "the content from the files", "and", "Apache Tika");
-
- @Test
- public void testOdtFileAsyncReader() throws Exception {
- doTestOdtFileReader(false);
- }
- @Test
- public void testOdtFileSyncReader() throws Exception {
- doTestOdtFileReader(true);
- }
- private void doTestOdtFileReader(boolean sync) throws Exception {
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
- TikaSource source = new TikaSource(TikaIO.read()
- .withParseSynchronously(sync)
- .from(resourcePath));
- TikaReader reader = (TikaReader) source.createReader(null);
-
- List<String> content = new LinkedList<String>();
- for (boolean available = reader.start(); available; available = reader.advance()) {
- content.add(reader.getCurrent());
- }
- assertTrue(content.containsAll(ODT_FILE));
- if (!sync) {
- assertNotNull(reader.getExecutorService());
- } else {
- assertNull(reader.getExecutorService());
- }
- reader.close();
- }
-
- @Test
- public void testOdtFilesReader() throws Exception {
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
- String filePattern = resourcePath.replace("apache-beam-tika1", "*");
-
- TikaSource source = new TikaSource(TikaIO.read().from(filePattern));
- TikaSource.FilePatternTikaReader reader =
- (TikaSource.FilePatternTikaReader) source.createReader(null);
- List<String> content = new LinkedList<String>();
- for (boolean available = reader.start(); available; available = reader.advance()) {
- content.add(reader.getCurrent());
- }
- assertTrue(content.containsAll(ODT_FILE));
- reader.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java
deleted file mode 100644
index 550f469..0000000
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.tika.TikaSource.TikaReader;
-import org.junit.Test;
-
-/**
- * Tests TikaSource.
- */
-public class TikaSourceTest {
-
- @Test
- public void testOdtFileSource() throws Exception {
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
- TikaSource source = new TikaSource(TikaIO.read().from(resourcePath));
- assertEquals(StringUtf8Coder.of(), source.getDefaultOutputCoder());
-
- assertEquals(TikaSource.Mode.FILEPATTERN, source.getMode());
- assertTrue(source.createReader(null) instanceof TikaReader);
-
- List<? extends TikaSource> sources = source.split(1, null);
- assertEquals(1, sources.size());
- TikaSource nextSource = sources.get(0);
- assertEquals(TikaSource.Mode.SINGLE_FILE, nextSource.getMode());
- assertEquals(resourcePath, nextSource.getSingleFileMetadata().resourceId().toString());
- }
-
- @Test
- public void testOdtFilesSource() throws Exception {
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
- String resourcePath2 = getClass().getResource("/apache-beam-tika2.odt").getPath();
- String filePattern = resourcePath.replace("apache-beam-tika1", "*");
-
- TikaSource source = new TikaSource(TikaIO.read().from(filePattern));
- assertEquals(StringUtf8Coder.of(), source.getDefaultOutputCoder());
-
- assertEquals(TikaSource.Mode.FILEPATTERN, source.getMode());
- assertTrue(source.createReader(null) instanceof TikaSource.FilePatternTikaReader);
-
- List<? extends TikaSource> sources = source.split(1, null);
- assertEquals(2, sources.size());
- TikaSource nextSource = sources.get(0);
- assertEquals(TikaSource.Mode.SINGLE_FILE, nextSource.getMode());
- String nextSourceResource = nextSource.getSingleFileMetadata().resourceId().toString();
- TikaSource nextSource2 = sources.get(1);
- assertEquals(TikaSource.Mode.SINGLE_FILE, nextSource2.getMode());
- String nextSourceResource2 = nextSource2.getSingleFileMetadata().resourceId().toString();
- assertTrue(nextSourceResource.equals(resourcePath) && nextSourceResource2.equals(resourcePath2)
- || nextSourceResource.equals(resourcePath2) && nextSourceResource2.equals(resourcePath));
- }
-}
[2/3] beam git commit: Adds ParseResult.failure()
Posted by jk...@apache.org.
Adds ParseResult.failure()
Also groups the tests into fewer methods providing the same coverage.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c76c518b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c76c518b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c76c518b
Branch: refs/heads/master
Commit: c76c518b62eee22334ff77fd15ae4eeae089e828
Parents: ba93dd3
Author: Eugene Kirpichov <ki...@google.com>
Authored: Wed Oct 25 16:57:36 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Oct 26 13:20:39 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/testing/SuccessOrFailure.java | 21 +-
.../beam/sdk/util/SerializableThrowable.java | 49 +++++
sdks/java/io/tika/pom.xml | 6 +
.../apache/beam/sdk/io/tika/ParseResult.java | 104 +++++++---
.../org/apache/beam/sdk/io/tika/TikaIO.java | 192 +++++++++--------
.../beam/sdk/io/tika/ParseResultTest.java | 54 ++++-
.../org/apache/beam/sdk/io/tika/TikaIOTest.java | 208 +++++++------------
.../src/test/resources/apache-beam-tika-pdf.zip | Bin 11685 -> 0 bytes
.../src/test/resources/apache-beam-tika.pdf | Bin 12392 -> 0 bytes
.../src/test/resources/apache-beam-tika1.odt | Bin 12540 -> 0 bytes
.../src/test/resources/apache-beam-tika2.odt | Bin 11412 -> 0 bytes
.../resources/valid/apache-beam-tika-pdf.zip | Bin 0 -> 11685 bytes
.../test/resources/valid/apache-beam-tika.odt | Bin 0 -> 12540 bytes
13 files changed, 351 insertions(+), 283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
index a63bbcc..bac4eb3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
@@ -18,34 +18,17 @@
package org.apache.beam.sdk.testing;
import com.google.common.base.MoreObjects;
-import java.io.IOException;
-import java.io.ObjectInputStream;
import java.io.Serializable;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.DefaultCoder;
import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.util.SerializableThrowable;
/**
* Output of {@link PAssert}. Passed to a conclude function to act upon.
*/
@DefaultCoder(SerializableCoder.class)
public final class SuccessOrFailure implements Serializable {
- private static final class SerializableThrowable implements Serializable {
- @Nullable private final Throwable throwable;
- @Nullable private final StackTraceElement[] stackTrace;
-
- private SerializableThrowable(@Nullable Throwable t) {
- this.throwable = t;
- this.stackTrace = (t == null) ? null : t.getStackTrace();
- }
-
- private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
- is.defaultReadObject();
- if (throwable != null) {
- throwable.setStackTrace(stackTrace);
- }
- }
- }
private final boolean isSuccess;
@Nullable
@@ -68,7 +51,7 @@ public final class SuccessOrFailure implements Serializable {
@Nullable
public AssertionError assertionError() {
- return site == null ? null : site.wrap(throwable.throwable);
+ return site == null ? null : site.wrap(throwable.getThrowable());
}
public static SuccessOrFailure success() {
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
new file mode 100644
index 0000000..4951958
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+
+/**
+ * A wrapper around {@link Throwable} that preserves the stack trace on serialization, unlike
+ * regular {@link Throwable}.
+ */
+public final class SerializableThrowable implements Serializable {
+ @Nullable private final Throwable throwable;
+ @Nullable private final StackTraceElement[] stackTrace;
+
+ public SerializableThrowable(@Nullable Throwable t) {
+ this.throwable = t;
+ this.stackTrace = (t == null) ? null : t.getStackTrace();
+ }
+
+ @Nullable
+ public Throwable getThrowable() {
+ return throwable;
+ }
+
+ private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
+ is.defaultReadObject();
+ if (throwable != null) {
+ throwable.setStackTrace(stackTrace);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/pom.xml b/sdks/java/io/tika/pom.xml
index d7f7e42..5e48099 100644
--- a/sdks/java/io/tika/pom.xml
+++ b/sdks/java/io/tika/pom.xml
@@ -67,6 +67,12 @@
</dependency>
<dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<classifier>tests</classifier>
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
index 0a77491..f78d603 100644
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
@@ -17,61 +17,95 @@
*/
package org.apache.beam.sdk.io.tika;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Throwables;
import java.io.Serializable;
import java.util.Arrays;
-
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.util.SerializableThrowable;
import org.apache.tika.metadata.Metadata;
/**
- * Tika parse result containing the file location, metadata
- * and content converted to String.
+ * The result of parsing a single file with Tika: contains the file's location, metadata, extracted
+ * text, and optionally an error. If there is an error, the metadata and extracted text may be
+ * partial (i.e. not represent the entire file).
*/
-@SuppressWarnings("serial")
public class ParseResult implements Serializable {
private final String fileLocation;
private final String content;
private final Metadata metadata;
private final String[] metadataNames;
+ @Nullable private final SerializableThrowable error;
+
+ public static ParseResult success(String fileLocation, String content, Metadata metadata) {
+ return new ParseResult(fileLocation, content, metadata, null);
+ }
- public ParseResult(String fileLocation, String content) {
- this(fileLocation, content, new Metadata());
+ public static ParseResult success(String fileLocation, String content) {
+ return new ParseResult(fileLocation, content, new Metadata(), null);
}
- public ParseResult(String fileLocation, String content, Metadata metadata) {
+ public static ParseResult failure(
+ String fileLocation, String partialContent, Metadata partialMetadata, Throwable error) {
+ return new ParseResult(fileLocation, partialContent, partialMetadata, error);
+ }
+
+ private ParseResult(String fileLocation, String content, Metadata metadata, Throwable error) {
+ checkArgument(fileLocation != null, "fileLocation can not be null");
+ checkArgument(content != null, "content can not be null");
+ checkArgument(metadata != null, "metadata can not be null");
this.fileLocation = fileLocation;
this.content = content;
this.metadata = metadata;
this.metadataNames = metadata.names();
+ this.error = (error == null) ? null : new SerializableThrowable(error);
+ }
+
+ /** Returns the absolute path to the input file. */
+ public String getFileLocation() {
+ return fileLocation;
+ }
+
+ /** Returns whether this file was parsed successfully. */
+ public boolean isSuccess() {
+ return error == null;
+ }
+
+ /** Returns the parse error, if the file was parsed unsuccessfully. */
+ public Throwable getError() {
+ checkState(error != null, "This is a successful ParseResult");
+ return error.getThrowable();
}
/**
- * Gets a file content.
+ * Same as {@link #getError}, but returns the complete stack trace of the error as a {@link
+ * String}.
*/
+ public String getErrorAsString() {
+ return Throwables.getStackTraceAsString(getError());
+ }
+
+ /** Returns the extracted text. May be partial, if this parse result contains a failure. */
public String getContent() {
return content;
}
- /**
- * Gets a file metadata.
- */
+ /** Returns the extracted metadata. May be partial, if this parse result contains a failure. */
public Metadata getMetadata() {
return metadata;
}
- /**
- * Gets a file location.
- */
- public String getFileLocation() {
- return fileLocation;
- }
-
@Override
public int hashCode() {
- int hashCode = 1;
- hashCode = 31 * hashCode + fileLocation.hashCode();
- hashCode = 31 * hashCode + content.hashCode();
- hashCode = 31 * hashCode + getMetadataHashCode();
- return hashCode;
+ return Objects.hash(
+ getFileLocation(),
+ getContent(),
+ getMetadataHashCode(),
+ isSuccess() ? "" : Throwables.getStackTraceAsString(getError()));
}
@Override
@@ -80,14 +114,16 @@ public class ParseResult implements Serializable {
return false;
}
- ParseResult pr = (ParseResult) obj;
- return this.fileLocation.equals(pr.fileLocation)
- && this.content.equals(pr.content)
- && this.metadata.equals(pr.metadata);
+ ParseResult other = (ParseResult) obj;
+ return Objects.equals(getFileLocation(), other.getFileLocation())
+ && Objects.equals(getContent(), other.getContent())
+ && Objects.equals(getMetadata(), other.getMetadata())
+ && (isSuccess()
+ ? other.isSuccess()
+ : (!other.isSuccess() && Objects.equals(getErrorAsString(), other.getErrorAsString())));
}
- //TODO:
- // Remove this function and use metadata.hashCode() once Apache Tika 1.17 gets released.
+ // TODO: Remove this function and use metadata.hashCode() once Apache Tika 1.17 gets released.
private int getMetadataHashCode() {
int hashCode = 0;
for (String name : metadataNames) {
@@ -95,4 +131,14 @@ public class ParseResult implements Serializable {
}
return hashCode;
}
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("fileLocation", fileLocation)
+ .add("content", "<" + content.length() + " chars>")
+ .add("metadata", metadata)
+ .add("error", getError() == null ? null : Throwables.getStackTraceAsString(getError()))
+ .toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
index 32353e1..26f116d 100644
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
@@ -17,15 +17,13 @@
*/
package org.apache.beam.sdk.io.tika;
+import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.auto.value.AutoValue;
-
import java.io.InputStream;
import java.nio.channels.Channels;
-
import javax.annotation.Nullable;
-
import org.apache.beam.sdk.annotations.Experimental;
import org.apache.beam.sdk.io.Compression;
import org.apache.beam.sdk.io.FileIO;
@@ -49,68 +47,60 @@ import org.apache.tika.parser.Parser;
import org.apache.tika.sax.ToTextContentHandler;
import org.xml.sax.ContentHandler;
-
-
-
/**
- * A collection of {@link PTransform} transforms for parsing arbitrary files using Apache Tika.
- * Files in many well known text, binary or scientific formats can be processed.
+ * Transforms for parsing arbitrary files using <a href="https://tika.apache.org/">Apache Tika</a>.
+ *
+ * <p>Tika is able to extract text and metadata from files in many well known text, binary and
+ * scientific formats.
*
- * <p>{@link TikaIO.Parse} and {@link TikaIO.ParseAll} parse the files and return
- * a {@link PCollection} containing one {@link ParseResult} per each file.
+ * <p>The entry points are {@link #parse} and {@link #parseFiles}. They parse a set of files and
+ * return a {@link PCollection} containing one {@link ParseResult} per each file. {@link #parse}
+ * implements the common case of parsing all files matching a single filepattern, while {@link
+ * #parseFiles} should be used for all use cases requiring more control, in combination with {@link
+ * FileIO#match} and {@link FileIO#readMatches} (see their respective documentation).
*
- * <p>Combine {@link TikaIO.ParseAll} with {@link FileIO.Match}
- * and {@link FileIO.ReadMatches} to match, read and parse the files.
+ * <p>{@link #parse} does not automatically uncompress compressed files: they are passed to Tika
+ * as-is.
*
- * <p>Example:
+ * <p>It's possible that some files will partially or completely fail to parse. In that case, the
+ * respective {@link ParseResult} will be marked unsuccessful (see {@link ParseResult#isSuccess})
+ * and will contain the error, available via {@link ParseResult#getError}.
+ *
+ * <p>Example: using {@link #parse} to parse all PDF files in a directory on GCS.
*
* <pre>{@code
* Pipeline p = ...;
*
- * // A simple parse of a local PDF file (only runs locally):
* PCollection<ParseResult> results =
- * p.apply(FileIO.match().filepattern("/local/path/to/file.pdf"))
- * .apply(FileIO.readMatches())
- * .apply(TikaIO.parseFiles());
+ * p.apply(TikaIO.parse().filepattern("gs://my-bucket/files/*.pdf"));
* }</pre>
*
- * <p>Use {@link TikaIO.Parse} to match, read and parse the files in simple cases.
- *
- * <p>Example:
+ * <p>Example: using {@link #parseFiles} in combination with {@link FileIO} to continuously parse
+ * new PDF files arriving into the directory.
*
* <pre>{@code
* Pipeline p = ...;
*
- * // A simple parse of a local PDF file (only runs locally):
* PCollection<ParseResult> results =
- * p.apply(TikaIO.parseAll().filepattern("/local/path/to/file.pdf"));
+ * p.apply(FileIO.match().filepattern("gs://my-bucket/files/*.pdf")
+ * .continuously(...))
+ * .apply(FileIO.readMatches())
+ * .apply(TikaIO.parseFiles());
* }</pre>
- *
- * <b>Warning:</b> the API of this IO is likely to change in the next release.
*/
@Experimental(Experimental.Kind.SOURCE_SINK)
public class TikaIO {
-
- /**
- * A {@link PTransform} that matches and parses the files
- * and returns a bounded {@link PCollection} of {@link ParseResult}.
- */
+ /** Parses files matching a given filepattern. */
public static Parse parse() {
- return new AutoValue_TikaIO_Parse.Builder()
- .build();
+ return new AutoValue_TikaIO_Parse.Builder().build();
}
- /**
- * A {@link PTransform} that accepts a {@link PCollection} of {@link ReadableFile}
- * and returns a {@link PCollection} of {@link ParseResult}.
- */
- public static ParseAll parseAll() {
- return new AutoValue_TikaIO_ParseAll.Builder()
- .build();
+ /** Parses files in a {@link PCollection} of {@link ReadableFile}. */
+ public static ParseFiles parseFiles() {
+ return new AutoValue_TikaIO_ParseFiles.Builder().build();
}
/** Implementation of {@link #parse}. */
- @SuppressWarnings("serial")
@AutoValue
public abstract static class Parse extends PTransform<PBegin, PCollection<ParseResult>> {
@Nullable
@@ -139,8 +129,7 @@ public class TikaIO {
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
- builder
- .addIfNotNull(
+ builder.addIfNotNull(
DisplayData.item("filePattern", getFilepattern()).withLabel("File Pattern"));
}
@@ -149,65 +138,68 @@ public class TikaIO {
return input
.apply(FileIO.match().filepattern(getFilepattern()))
.apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
- .apply(parseAll());
+ .apply(parseFiles());
}
}
- /** Implementation of {@link #parseAll}. */
- @SuppressWarnings("serial")
+ /** Implementation of {@link #parseFiles}. */
@AutoValue
- public abstract static class ParseAll extends
- PTransform<PCollection<ReadableFile>, PCollection<ParseResult>> {
+ public abstract static class ParseFiles
+ extends PTransform<PCollection<ReadableFile>, PCollection<ParseResult>> {
+
+ @Nullable
+ abstract ValueProvider<String> getTikaConfigPath();
- @Nullable abstract ValueProvider<String> getTikaConfigPath();
- @Nullable abstract Metadata getInputMetadata();
+ @Nullable
+ abstract String getContentTypeHint();
+
+ @Nullable
+ abstract Metadata getInputMetadata();
abstract Builder toBuilder();
@AutoValue.Builder
abstract static class Builder {
abstract Builder setTikaConfigPath(ValueProvider<String> tikaConfigPath);
+
+ abstract Builder setContentTypeHint(String contentTypeHint);
+
abstract Builder setInputMetadata(Metadata metadata);
- abstract ParseAll build();
+ abstract ParseFiles build();
}
/**
- * Returns a new transform which will use the custom TikaConfig.
+ * Uses the given <a
+ * href="https://tika.apache.org/1.16/configuring.html#Using_a_Tika_Configuration_XML_file">Tika
+ * Configuration XML file</a>.
*/
- public ParseAll withTikaConfigPath(String tikaConfigPath) {
- checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
+ public ParseFiles withTikaConfigPath(String tikaConfigPath) {
+ checkArgument(tikaConfigPath != null, "tikaConfigPath can not be null.");
return withTikaConfigPath(StaticValueProvider.of(tikaConfigPath));
}
- /** Same as {@code with(tikaConfigPath)}, but accepting a {@link ValueProvider}. */
- public ParseAll withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
- checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
- return toBuilder()
- .setTikaConfigPath(tikaConfigPath)
- .build();
+ /** Like {@code with(tikaConfigPath)}. */
+ public ParseFiles withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
+ checkArgument(tikaConfigPath != null, "tikaConfigPath can not be null.");
+ return toBuilder().setTikaConfigPath(tikaConfigPath).build();
}
/**
- * Returns a new transform which will use the provided content type hint
- * to make the file parser detection more efficient.
+ * Sets a content type hint to make the file parser detection more efficient. Overrides the
+ * content type hint in {@link #withInputMetadata}, if any.
*/
- public ParseAll withContentTypeHint(String contentType) {
- checkNotNull(contentType, "ContentType cannot be empty.");
- Metadata metadata = new Metadata();
- metadata.add(Metadata.CONTENT_TYPE, contentType);
- return withInputMetadata(metadata);
+ public ParseFiles withContentTypeHint(String contentTypeHint) {
+ checkNotNull(contentTypeHint, "contentTypeHint can not be null.");
+ return toBuilder().setContentTypeHint(contentTypeHint).build();
}
- /**
- * Returns a new transform which will use the provided input metadata
- * for parsing the files.
- */
- public ParseAll withInputMetadata(Metadata metadata) {
+ /** Sets the input metadata for {@link Parser#parse}. */
+ public ParseFiles withInputMetadata(Metadata metadata) {
Metadata inputMetadata = this.getInputMetadata();
if (inputMetadata != null) {
for (String name : metadata.names()) {
- inputMetadata.set(name, metadata.get(name));
+ inputMetadata.set(name, metadata.get(name));
}
} else {
inputMetadata = metadata;
@@ -225,27 +217,25 @@ public class TikaIO {
super.populateDisplayData(builder);
if (getTikaConfigPath() != null) {
- String tikaConfigPathDisplay = getTikaConfigPath().isAccessible()
- ? getTikaConfigPath().get() : getTikaConfigPath().toString();
- builder.add(DisplayData.item("tikaConfigPath", tikaConfigPathDisplay)
- .withLabel("TikaConfig Path"));
+ builder.add(
+ DisplayData.item("tikaConfigPath", getTikaConfigPath()).withLabel("TikaConfig Path"));
}
Metadata metadata = getInputMetadata();
if (metadata != null) {
//TODO: use metadata.toString() only without a trim() once Apache Tika 1.17 gets released
- builder
- .add(DisplayData.item("inputMetadata", metadata.toString().trim())
- .withLabel("Input Metadata"));
+ builder.add(
+ DisplayData.item("inputMetadata", metadata.toString().trim())
+ .withLabel("Input Metadata"));
}
+ builder.addIfNotNull(
+ DisplayData.item("contentTypeHint", getContentTypeHint()).withLabel("Content type hint"));
}
private static class ParseToStringFn extends DoFn<ReadableFile, ParseResult> {
+ private final ParseFiles spec;
+ private transient TikaConfig tikaConfig;
- private static final long serialVersionUID = 6837207505313720989L;
- private final TikaIO.ParseAll spec;
- private TikaConfig tikaConfig;
-
- ParseToStringFn(TikaIO.ParseAll spec) {
+ ParseToStringFn(ParseFiles spec) {
this.spec = spec;
}
@@ -254,8 +244,7 @@ public class TikaIO {
if (spec.getTikaConfigPath() != null) {
ResourceId configResource =
FileSystems.matchSingleFileSpec(spec.getTikaConfigPath().get()).resourceId();
- tikaConfig = new TikaConfig(
- Channels.newInputStream(FileSystems.open(configResource)));
+ tikaConfig = new TikaConfig(Channels.newInputStream(FileSystems.open(configResource)));
}
}
@@ -264,21 +253,30 @@ public class TikaIO {
ReadableFile file = c.element();
InputStream stream = Channels.newInputStream(file.open());
try (InputStream tikaStream = TikaInputStream.get(stream)) {
+ Parser parser =
+ tikaConfig == null ? new AutoDetectParser() : new AutoDetectParser(tikaConfig);
- final Parser parser = tikaConfig == null
- ? new AutoDetectParser() : new AutoDetectParser(tikaConfig);
-
- final ParseContext context = new ParseContext();
+ ParseContext context = new ParseContext();
context.set(Parser.class, parser);
- Metadata tikaMetadata = spec.getInputMetadata() != null
- ? spec.getInputMetadata() : new org.apache.tika.metadata.Metadata();
-
+ Metadata tikaMetadata =
+ spec.getInputMetadata() != null
+ ? spec.getInputMetadata()
+ : new org.apache.tika.metadata.Metadata();
+ if (spec.getContentTypeHint() != null) {
+ tikaMetadata.set(Metadata.CONTENT_TYPE, spec.getContentTypeHint());
+ }
+
+ String location = file.getMetadata().resourceId().toString();
+ ParseResult res;
ContentHandler tikaHandler = new ToTextContentHandler();
- parser.parse(tikaStream, tikaHandler, tikaMetadata, context);
-
- c.output(new ParseResult(file.getMetadata().resourceId().toString(),
- tikaHandler.toString(),
- tikaMetadata));
+ try {
+ parser.parse(tikaStream, tikaHandler, tikaMetadata, context);
+ res = ParseResult.success(location, tikaHandler.toString(), tikaMetadata);
+ } catch (Exception e) {
+ res = ParseResult.failure(location, tikaHandler.toString(), tikaMetadata, e);
+ }
+
+ c.output(res);
}
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
index fd86152..95bcee0 100644
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
@@ -17,20 +17,60 @@
*/
package org.apache.beam.sdk.io.tika;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
import org.apache.tika.metadata.Metadata;
import org.junit.Test;
-/**
- * Tests ParseResult.
- */
+/** Tests {@link ParseResult}. */
public class ParseResultTest {
@Test
public void testEqualsAndHashCode() {
- ParseResult p1 = new ParseResult("a.txt", "hello", getMetadata());
- ParseResult p2 = new ParseResult("a.txt", "hello", getMetadata());
- assertEquals(p1, p2);
- assertEquals(p1.hashCode(), p2.hashCode());
+ ParseResult successBase = ParseResult.success("a.txt", "hello", getMetadata());
+ ParseResult successSame = ParseResult.success("a.txt", "hello", getMetadata());
+ ParseResult successDifferentName = ParseResult.success("b.txt", "hello", getMetadata());
+ ParseResult successDifferentContent = ParseResult.success("a.txt", "goodbye", getMetadata());
+ ParseResult successDifferentMetadata = ParseResult.success("a.txt", "hello", new Metadata());
+
+ RuntimeException oops = new RuntimeException("oops");
+ ParseResult failureBase = ParseResult.failure("a.txt", "", new Metadata(), oops);
+ ParseResult failureSame = ParseResult.failure("a.txt", "", new Metadata(), oops);
+ ParseResult failureDifferentName = ParseResult.failure("b.txt", "", new Metadata(), oops);
+ ParseResult failureDifferentContent =
+ ParseResult.failure("b.txt", "partial", new Metadata(), oops);
+ ParseResult failureDifferentMetadata = ParseResult.failure("b.txt", "", getMetadata(), oops);
+ ParseResult failureDifferentError =
+ ParseResult.failure("a.txt", "", new Metadata(), new RuntimeException("eek"));
+
+ assertEquals(successBase, successSame);
+ assertEquals(successBase.hashCode(), successSame.hashCode());
+
+ assertThat(successDifferentName, not(equalTo(successBase)));
+ assertThat(successDifferentContent, not(equalTo(successBase)));
+ assertThat(successDifferentMetadata, not(equalTo(successBase)));
+
+ assertThat(successDifferentName.hashCode(), not(equalTo(successBase.hashCode())));
+ assertThat(successDifferentContent.hashCode(), not(equalTo(successBase.hashCode())));
+ assertThat(successDifferentMetadata.hashCode(), not(equalTo(successBase.hashCode())));
+
+ assertThat(failureBase, not(equalTo(successBase)));
+ assertThat(successBase, not(equalTo(failureBase)));
+
+ assertEquals(failureBase, failureSame);
+ assertEquals(failureBase.hashCode(), failureSame.hashCode());
+
+ assertThat(failureDifferentName, not(equalTo(failureBase)));
+ assertThat(failureDifferentError, not(equalTo(failureBase)));
+ assertThat(failureDifferentContent, not(equalTo(failureBase)));
+ assertThat(failureDifferentMetadata, not(equalTo(failureBase)));
+
+ assertThat(failureDifferentName.hashCode(), not(equalTo(failureBase.hashCode())));
+ assertThat(failureDifferentError.hashCode(), not(equalTo(failureBase.hashCode())));
+ assertThat(failureDifferentContent.hashCode(), not(equalTo(failureBase.hashCode())));
+ assertThat(failureDifferentMetadata.hashCode(), not(equalTo(failureBase.hashCode())));
}
static Metadata getMetadata() {
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
index a985b0a..1c95e9f 100644
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
@@ -18,124 +18,109 @@
package org.apache.beam.sdk.io.tika;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
-import static org.hamcrest.Matchers.isA;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
-
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
import org.apache.beam.sdk.io.Compression;
import org.apache.beam.sdk.io.FileIO;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.values.PCollection;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-/**
- * Tests TikaInput.
- */
-public class TikaIOTest {
- private static final String PDF_FILE =
- "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
- + "Combining\n\nApache Beam\n\nand\n\nApache Tika\n\ncan help to ingest\n\n"
- + "the content from the files\n\nin most known formats.\n\n\n";
+/** Tests for {@link TikaIO}. */
+public class TikaIOTest implements Serializable {
private static final String PDF_ZIP_FILE =
"\n\n\n\n\n\n\n\napache-beam-tika.pdf\n\n\nCombining\n\n\nApache Beam\n\n\n"
- + "and\n\n\nApache Tika\n\n\ncan help to ingest\n\n\nthe content from the files\n\n\n"
- + "in most known formats.\n\n\n\n\n\n\n";
+ + "and\n\n\nApache Tika\n\n\ncan help to ingest\n\n\nthe content from the files\n\n\n"
+ + "in most known formats.\n\n\n\n\n\n\n";
private static final String ODT_FILE =
"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
- + "Combining\nApache Beam\nand\nApache Tika\ncan help to ingest\nthe content from the"
- + " files\nin most known formats.\n";
-
- private static final String ODT_FILE2 =
- "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
- + "Open Office\nPDF\nExcel\nText\nScientific\nand other formats\nare supported.\n";
+ + "Combining\nApache Beam\nand\nApache Tika\ncan help to ingest\nthe content from the"
+ + " files\nin most known formats.\n";
@Rule
- public TestPipeline p = TestPipeline.create();
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void testParsePdfFile() throws IOException {
-
- String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
-
- doTestParse(resourcePath, new ParseResult(resourcePath, PDF_FILE));
- }
-
- private void doTestParse(String resourcePath, ParseResult... expectedResults)
- throws IOException {
- PCollection<ParseResult> output =
- p.apply("ParseAll", TikaIO.parse().filepattern(resourcePath))
- .apply(ParDo.of(new FilterMetadataFn()));
- PAssert.that(output).containsInAnyOrder(expectedResults);
- p.run();
- }
-
- @Test
- public void testParseAllPdfFile() throws IOException {
-
- String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
-
- doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_FILE));
- }
-
- @Test
- public void testParseAllZipPdfFile() throws IOException {
-
- String resourcePath = getClass().getResource("/apache-beam-tika-pdf.zip").getPath();
-
- doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_ZIP_FILE));
- }
+ public transient TestPipeline p = TestPipeline.create();
@Test
- public void testParseAllOdtFile() throws IOException {
-
- String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-
- doTestParseAll(resourcePath, new ParseResult(resourcePath, ODT_FILE, getOdtMetadata()));
+ public void testParseAndParseFiles() throws IOException {
+ Path root =
+ Paths.get(getClass().getResource("/valid/apache-beam-tika.odt").getPath()).getParent();
+
+ List<ParseResult> expected =
+ Arrays.asList(
+ ParseResult.success(
+ root.resolve("apache-beam-tika.odt").toString(), ODT_FILE, getOdtMetadata()),
+ ParseResult.success(root.resolve("apache-beam-tika-pdf.zip").toString(), PDF_ZIP_FILE));
+
+ PCollection<ParseResult> parse =
+ p.apply("Parse", TikaIO.parse().filepattern(root.resolve("*").toString()))
+ .apply("FilterParse", ParDo.of(new FilterMetadataFn()));
+ PAssert.that(parse).containsInAnyOrder(expected);
+
+ PCollection<ParseResult> parseFiles =
+ p.apply("ParseFiles", FileIO.match().filepattern(root.resolve("*").toString()))
+ .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+ .apply(TikaIO.parseFiles())
+ .apply("FilterParseFiles", ParDo.of(new FilterMetadataFn()));
+ PAssert.that(parseFiles).containsInAnyOrder(expected);
+ p.run();
}
- @Test
- public void testParseAllOdtFiles() throws IOException {
- String resourcePath1 = getClass().getResource("/apache-beam-tika1.odt").getPath();
- String resourcePath2 = getClass().getResource("/apache-beam-tika2.odt").getPath();
- String resourcePath = resourcePath1.replace("apache-beam-tika1", "*");
-
- doTestParseAll(resourcePath, new ParseResult(resourcePath1, ODT_FILE, getOdtMetadata()),
- new ParseResult(resourcePath2, ODT_FILE2));
+ private static Metadata getOdtMetadata() {
+ Metadata m = new Metadata();
+ m.set("Author", "BeamTikaUser");
+ return m;
}
- private void doTestParseAll(String resourcePath, ParseResult... expectedResults)
- throws IOException {
- PCollection<ParseResult> output =
- p.apply("ParseFiles", FileIO.match().filepattern(resourcePath))
- .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
- .apply(TikaIO.parseAll())
- .apply(ParDo.of(new FilterMetadataFn()));
- PAssert.that(output).containsInAnyOrder(expectedResults);
- p.run();
+ private static class FilterMetadataFn extends DoFn<ParseResult, ParseResult> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ ParseResult result = c.element();
+ Metadata m = new Metadata();
+ // Files contain many metadata properties. This function drops all but the "Author"
+ // property manually added to "apache-beam-tika.odt" resource only to make
+ // the tests simpler
+ if (result.getFileLocation().endsWith("valid/apache-beam-tika.odt")) {
+ m.set("Author", result.getMetadata().get("Author"));
+ }
+ ParseResult newResult = ParseResult.success(result.getFileLocation(), result.getContent(), m);
+ c.output(newResult);
+ }
}
@Test
- public void testParseAllDamagedPdfFile() throws IOException {
- thrown.expectCause(isA(TikaException.class));
- String resourcePath = getClass().getResource("/damaged.pdf").getPath();
-
- p.apply("ParseInvalidPdfFile", FileIO.match().filepattern(resourcePath))
- .apply(FileIO.readMatches())
- .apply(TikaIO.parseAll());
+ public void testParseDamagedPdfFile() throws IOException {
+ String path = getClass().getResource("/damaged.pdf").getPath();
+ PCollection<ParseResult> res = p.apply("ParseInvalidPdfFile", TikaIO.parse().filepattern(path));
+
+ PAssert.thatSingleton(res)
+ .satisfies(
+ new SerializableFunction<ParseResult, Void>() {
+ @Override
+ public Void apply(ParseResult input) {
+ assertEquals(path, input.getFileLocation());
+ assertFalse(input.isSuccess());
+ assertTrue(input.getError() instanceof TikaException);
+ return null;
+ }
+ });
p.run();
}
@@ -150,54 +135,15 @@ public class TikaIOTest {
}
@Test
- public void testParseAllDisplayData() {
- TikaIO.ParseAll parseAll = TikaIO.parseAll()
- .withTikaConfigPath("tikaconfigpath")
- .withContentTypeHint("application/pdf");
-
- DisplayData displayData = DisplayData.from(parseAll);
-
- assertThat(displayData, hasDisplayItem("tikaConfigPath", "tikaconfigpath"));
- assertThat(displayData, hasDisplayItem("inputMetadata",
- "Content-Type=application/pdf"));
- assertEquals(2, displayData.items().size());
- }
-
- @Test
- public void testParseAllDisplayDataWithCustomOptions() {
- TikaIO.ParseAll parseAll = TikaIO.parseAll()
- .withTikaConfigPath("/tikaConfigPath")
- .withContentTypeHint("application/pdf");
+ public void testParseFilesDisplayData() {
+ TikaIO.ParseFiles parseFiles =
+ TikaIO.parseFiles()
+ .withTikaConfigPath("/tikaConfigPath")
+ .withContentTypeHint("application/pdf");
- DisplayData displayData = DisplayData.from(parseAll);
+ DisplayData displayData = DisplayData.from(parseFiles);
assertThat(displayData, hasDisplayItem("tikaConfigPath", "/tikaConfigPath"));
- assertThat(displayData, hasDisplayItem("inputMetadata",
- "Content-Type=application/pdf"));
- assertEquals(2, displayData.items().size());
- }
-
- static class FilterMetadataFn extends DoFn<ParseResult, ParseResult> {
- private static final long serialVersionUID = 6338014219600516621L;
-
- @ProcessElement
- public void processElement(ProcessContext c) {
- ParseResult result = c.element();
- Metadata m = new Metadata();
- // Files contain many metadata properties. This function drops all but the "Author"
- // property manually added to "apache-beam-tika1.odt" resource only to make
- // the tests simpler
- if (result.getFileLocation().endsWith("apache-beam-tika1.odt")) {
- m.set("Author", result.getMetadata().get("Author"));
- }
- ParseResult newResult = new ParseResult(result.getFileLocation(), result.getContent(), m);
- c.output(newResult);
- }
- }
-
- static Metadata getOdtMetadata() {
- Metadata m = new Metadata();
- m.set("Author", "BeamTikaUser");
- return m;
+ assertThat(displayData, hasDisplayItem("contentTypeHint", "application/pdf"));
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip b/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip
deleted file mode 100644
index 4c0e0ef..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip and /dev/null differ
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf b/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf
deleted file mode 100644
index d3c7f14..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf and /dev/null differ
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt b/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt
deleted file mode 100644
index 87c5577..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt and /dev/null differ
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt b/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt
deleted file mode 100644
index a0ff320..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt and /dev/null differ
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip
new file mode 100644
index 0000000..4c0e0ef
Binary files /dev/null and b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip differ
http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt
new file mode 100644
index 0000000..87c5577
Binary files /dev/null and b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt differ
[3/3] beam git commit: This closes #3835: [BEAM-2994] Many
improvements to TikaIO
Posted by jk...@apache.org.
This closes #3835: [BEAM-2994] Many improvements to TikaIO
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0c221137
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0c221137
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0c221137
Branch: refs/heads/master
Commit: 0c2211375d1fc4abebda7aa7cf7960e7b09fdc1b
Parents: e01c78d c76c518
Author: Eugene Kirpichov <ki...@google.com>
Authored: Thu Oct 26 13:21:07 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Oct 26 13:21:07 2017 -0700
----------------------------------------------------------------------
.../beam/sdk/testing/SuccessOrFailure.java | 21 +-
.../beam/sdk/util/SerializableThrowable.java | 49 ++
sdks/java/io/tika/pom.xml | 16 +-
.../apache/beam/sdk/io/tika/ParseResult.java | 144 ++++++
.../org/apache/beam/sdk/io/tika/TikaIO.java | 394 ++++++++--------
.../apache/beam/sdk/io/tika/TikaOptions.java | 78 ----
.../org/apache/beam/sdk/io/tika/TikaSource.java | 466 -------------------
.../beam/sdk/io/tika/ParseResultTest.java | 83 ++++
.../org/apache/beam/sdk/io/tika/TikaIOTest.java | 290 ++++--------
.../apache/beam/sdk/io/tika/TikaReaderTest.java | 82 ----
.../apache/beam/sdk/io/tika/TikaSourceTest.java | 73 ---
.../src/test/resources/apache-beam-tika-pdf.zip | Bin 11685 -> 0 bytes
.../src/test/resources/apache-beam-tika.pdf | Bin 12392 -> 0 bytes
.../src/test/resources/apache-beam-tika1.odt | Bin 12540 -> 0 bytes
.../src/test/resources/apache-beam-tika2.odt | Bin 11412 -> 0 bytes
.../resources/valid/apache-beam-tika-pdf.zip | Bin 0 -> 11685 bytes
.../test/resources/valid/apache-beam-tika.odt | Bin 0 -> 12540 bytes
17 files changed, 556 insertions(+), 1140 deletions(-)
----------------------------------------------------------------------