You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jk...@apache.org on 2017/10/26 20:21:30 UTC

[1/3] beam git commit: Many improvements to TikaIO

Repository: beam
Updated Branches:
  refs/heads/master e01c78da7 -> 0c2211375


Many improvements to TikaIO

This addresses most of the comments in #3378.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ba93dd39
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ba93dd39
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ba93dd39

Branch: refs/heads/master
Commit: ba93dd39111ab2b13f811d0abeb76a49a4a4f035
Parents: e01c78d
Author: Sergey Beryozkin <sb...@gmail.com>
Authored: Mon Sep 11 16:11:10 2017 +0100
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Oct 26 12:45:19 2017 -0700

----------------------------------------------------------------------
 sdks/java/io/tika/pom.xml                       |  10 -
 .../apache/beam/sdk/io/tika/ParseResult.java    |  98 ++++
 .../org/apache/beam/sdk/io/tika/TikaIO.java     | 334 ++++++-------
 .../apache/beam/sdk/io/tika/TikaOptions.java    |  78 ----
 .../org/apache/beam/sdk/io/tika/TikaSource.java | 466 -------------------
 .../beam/sdk/io/tika/ParseResultTest.java       |  43 ++
 .../org/apache/beam/sdk/io/tika/TikaIOTest.java | 252 ++++------
 .../apache/beam/sdk/io/tika/TikaReaderTest.java |  82 ----
 .../apache/beam/sdk/io/tika/TikaSourceTest.java |  73 ---
 9 files changed, 392 insertions(+), 1044 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/pom.xml b/sdks/java/io/tika/pom.xml
index b8f7ece..d7f7e42 100644
--- a/sdks/java/io/tika/pom.xml
+++ b/sdks/java/io/tika/pom.xml
@@ -54,16 +54,6 @@
         </dependency>
 
         <dependency>
-            <groupId>joda-time</groupId>
-            <artifactId>joda-time</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-        </dependency>
-
-        <dependency>
             <groupId>org.apache.tika</groupId>
             <artifactId>tika-core</artifactId>
             <version>${tika.version}</version>

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
new file mode 100644
index 0000000..0a77491
--- /dev/null
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+import org.apache.tika.metadata.Metadata;
+
+/**
+ * Tika parse result containing the file location, metadata
+ * and content converted to String.
+ */
+@SuppressWarnings("serial")
+public class ParseResult implements Serializable {
+  private final String fileLocation;
+  private final String content;
+  private final Metadata metadata;
+  private final String[] metadataNames;
+
+  public ParseResult(String fileLocation, String content) {
+    this(fileLocation, content, new Metadata());
+  }
+
+  public ParseResult(String fileLocation, String content, Metadata metadata) {
+    this.fileLocation = fileLocation;
+    this.content = content;
+    this.metadata = metadata;
+    this.metadataNames = metadata.names();
+  }
+
+  /**
+   * Gets a file content.
+   */
+  public String getContent() {
+    return content;
+  }
+
+  /**
+   * Gets a file metadata.
+   */
+  public Metadata getMetadata() {
+    return metadata;
+  }
+
+  /**
+   * Gets a file location.
+   */
+  public String getFileLocation() {
+    return fileLocation;
+  }
+
+  @Override
+  public int hashCode() {
+    int hashCode = 1;
+    hashCode = 31 * hashCode + fileLocation.hashCode();
+    hashCode = 31 * hashCode + content.hashCode();
+    hashCode = 31 * hashCode + getMetadataHashCode();
+    return hashCode;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (!(obj instanceof ParseResult)) {
+      return false;
+    }
+
+    ParseResult pr = (ParseResult) obj;
+    return this.fileLocation.equals(pr.fileLocation)
+      && this.content.equals(pr.content)
+      && this.metadata.equals(pr.metadata);
+  }
+
+  //TODO:
+  // Remove this function and use metadata.hashCode() once Apache Tika 1.17 gets released.
+  private int getMetadataHashCode() {
+    int hashCode = 0;
+    for (String name : metadataNames) {
+      hashCode += name.hashCode() ^ Arrays.hashCode(metadata.getValues(name));
+    }
+    return hashCode;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
index 4876dcf..32353e1 100644
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
@@ -18,40 +18,72 @@
 package org.apache.beam.sdk.io.tika;
 
 import static com.google.common.base.Preconditions.checkNotNull;
+
 import com.google.auto.value.AutoValue;
 
+import java.io.InputStream;
+import java.nio.channels.Channels;
+
 import javax.annotation.Nullable;
+
 import org.apache.beam.sdk.annotations.Experimental;
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.Read.Bounded;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
+import org.apache.beam.sdk.io.FileIO.ReadableFile;
+import org.apache.beam.sdk.io.FileSystems;
+import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.io.TikaInputStream;
 import org.apache.tika.metadata.Metadata;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.ToTextContentHandler;
+import org.xml.sax.ContentHandler;
+
+
 
 
 /**
- * {@link PTransform} for parsing arbitrary files using Apache Tika.
+ * A collection of {@link PTransform} transforms for parsing arbitrary files using Apache Tika.
  * Files in many well known text, binary or scientific formats can be processed.
  *
- * <p>To read a {@link PCollection} from one or more files
- * use {@link TikaIO.Read#from(String)}
- * to specify the path of the file(s) to be read.
+ * <p>{@link TikaIO.Parse} and {@link TikaIO.ParseAll} parse the files and return
+ * a {@link PCollection} containing one {@link ParseResult} per each file.
+ *
+ * <p>Combine {@link TikaIO.ParseAll} with {@link FileIO.Match}
+ * and {@link FileIO.ReadMatches} to match, read and parse the files.
+ *
+ * <p>Example:
  *
- * <p>{@link TikaIO.Read} returns a bounded {@link PCollection} of {@link String Strings},
- * each corresponding to a sequence of characters reported by Apache Tika SAX Parser.
+ * <pre>{@code
+ * Pipeline p = ...;
+ *
+ * // A simple parse of a local PDF file (only runs locally):
+ * PCollection<ParseResult> results =
+ *   p.apply(FileIO.match().filepattern("/local/path/to/file.pdf"))
+ *    .apply(FileIO.readMatches())
+ *    .apply(TikaIO.parseFiles());
+ * }</pre>
+ *
+ * <p>Use {@link TikaIO.Parse} to match, read and parse the files in simple cases.
  *
  * <p>Example:
  *
  * <pre>{@code
  * Pipeline p = ...;
  *
- * // A simple Read of a local PDF file (only runs locally):
- * PCollection<String> content = p.apply(TikaInput.from("/local/path/to/file.pdf"));
+ * // A simple parse of a local PDF file (only runs locally):
+ * PCollection<ParseResult> results =
+ *   p.apply(TikaIO.parseAll().filepattern("/local/path/to/file.pdf"));
  * }</pre>
  *
  * <b>Warning:</b> the API of this IO is likely to change in the next release.
@@ -60,85 +92,96 @@ import org.apache.tika.metadata.Metadata;
 public class TikaIO {
 
   /**
-   * A {@link PTransform} that parses one or more files and returns a bounded {@link PCollection}
-   * containing one element for each sequence of characters reported by Apache Tika SAX Parser.
+   * A {@link PTransform} that matches and parses the files
+   * and returns a bounded {@link PCollection} of {@link ParseResult}.
    */
-   public static Read read() {
-     return new AutoValue_TikaIO_Read.Builder()
-        .setQueuePollTime(Read.DEFAULT_QUEUE_POLL_TIME)
-        .setQueueMaxPollTime(Read.DEFAULT_QUEUE_MAX_POLL_TIME)
+  public static Parse parse() {
+    return new AutoValue_TikaIO_Parse.Builder()
         .build();
-   }
+  }
 
-   /** Implementation of {@link #read}. */
-  @AutoValue
-  public abstract static class Read extends PTransform<PBegin, PCollection<String>> {
-    private static final long serialVersionUID = 2198301984784351829L;
-    public static final long DEFAULT_QUEUE_POLL_TIME = 50L;
-    public static final long DEFAULT_QUEUE_MAX_POLL_TIME = 3000L;
+  /**
+   * A {@link PTransform} that accepts a {@link PCollection} of {@link ReadableFile}
+   * and returns a {@link PCollection} of {@link ParseResult}.
+   */
+  public static ParseAll parseAll() {
+    return new AutoValue_TikaIO_ParseAll.Builder()
+        .build();
+  }
 
-    @Nullable abstract ValueProvider<String> getFilepattern();
-    @Nullable abstract ValueProvider<String> getTikaConfigPath();
-    @Nullable abstract Metadata getInputMetadata();
-    @Nullable abstract Boolean getReadOutputMetadata();
-    @Nullable abstract Long getQueuePollTime();
-    @Nullable abstract Long getQueueMaxPollTime();
-    @Nullable abstract Integer getMinimumTextLength();
-    @Nullable abstract Boolean getParseSynchronously();
+  /** Implementation of {@link #parse}. */
+  @SuppressWarnings("serial")
+  @AutoValue
+  public abstract static class Parse extends PTransform<PBegin, PCollection<ParseResult>> {
+    @Nullable
+    abstract ValueProvider<String> getFilepattern();
 
     abstract Builder toBuilder();
 
     @AutoValue.Builder
     abstract static class Builder {
       abstract Builder setFilepattern(ValueProvider<String> filepattern);
-      abstract Builder setTikaConfigPath(ValueProvider<String> tikaConfigPath);
-      abstract Builder setInputMetadata(Metadata metadata);
-      abstract Builder setReadOutputMetadata(Boolean value);
-      abstract Builder setQueuePollTime(Long value);
-      abstract Builder setQueueMaxPollTime(Long value);
-      abstract Builder setMinimumTextLength(Integer value);
-      abstract Builder setParseSynchronously(Boolean value);
 
-      abstract Read build();
+      abstract Parse build();
     }
 
-    /**
-     * A {@link PTransform} that parses one or more files with the given filename
-     * or filename pattern and returns a bounded {@link PCollection} containing
-     * one element for each sequence of characters reported by Apache Tika SAX Parser.
-     *
-     * <p>Filepattern can be a local path (if running locally), or a Google Cloud Storage
-     * filename or filename pattern of the form {@code "gs://<bucket>/<filepath>"}
-     * (if running locally or using remote execution service).
-     *
-     * <p>Standard <a href="http://docs.oracle.com/javase/tutorial/essential/io/find.html" >Java
-     * Filesystem glob patterns</a> ("*", "?", "[..]") are supported.
-     */
-    public Read from(String filepattern) {
-      checkNotNull(filepattern, "Filepattern cannot be empty.");
-      return from(StaticValueProvider.of(filepattern));
+    /** Matches the given filepattern. */
+    public Parse filepattern(String filepattern) {
+      return this.filepattern(ValueProvider.StaticValueProvider.of(filepattern));
     }
 
-    /** Same as {@code from(filepattern)}, but accepting a {@link ValueProvider}. */
-    public Read from(ValueProvider<String> filepattern) {
-      checkNotNull(filepattern, "Filepattern cannot be empty.");
-      return toBuilder()
-          .setFilepattern(filepattern)
-          .setQueuePollTime(Read.DEFAULT_QUEUE_POLL_TIME)
-          .setQueueMaxPollTime(Read.DEFAULT_QUEUE_MAX_POLL_TIME)
-          .build();
+    /** Like {@link #filepattern(String)} but using a {@link ValueProvider}. */
+    public Parse filepattern(ValueProvider<String> filepattern) {
+      return toBuilder().setFilepattern(filepattern).build();
+    }
+
+    @Override
+    public void populateDisplayData(DisplayData.Builder builder) {
+      super.populateDisplayData(builder);
+
+      builder
+        .addIfNotNull(
+          DisplayData.item("filePattern", getFilepattern()).withLabel("File Pattern"));
+    }
+
+    @Override
+    public PCollection<ParseResult> expand(PBegin input) {
+      return input
+          .apply(FileIO.match().filepattern(getFilepattern()))
+          .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+          .apply(parseAll());
+    }
+  }
+
+  /** Implementation of {@link #parseAll}. */
+  @SuppressWarnings("serial")
+  @AutoValue
+  public abstract static class ParseAll extends
+    PTransform<PCollection<ReadableFile>, PCollection<ParseResult>> {
+
+    @Nullable abstract ValueProvider<String> getTikaConfigPath();
+    @Nullable abstract Metadata getInputMetadata();
+
+    abstract Builder toBuilder();
+
+    @AutoValue.Builder
+    abstract static class Builder {
+      abstract Builder setTikaConfigPath(ValueProvider<String> tikaConfigPath);
+      abstract Builder setInputMetadata(Metadata metadata);
+
+      abstract ParseAll build();
     }
 
     /**
      * Returns a new transform which will use the custom TikaConfig.
      */
-    public Read withTikaConfigPath(String tikaConfigPath) {
+    public ParseAll withTikaConfigPath(String tikaConfigPath) {
       checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
       return withTikaConfigPath(StaticValueProvider.of(tikaConfigPath));
     }
 
     /** Same as {@code with(tikaConfigPath)}, but accepting a {@link ValueProvider}. */
-    public Read withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
+    public ParseAll withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
       checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
       return toBuilder()
           .setTikaConfigPath(tikaConfigPath)
@@ -149,7 +192,7 @@ public class TikaIO {
      * Returns a new transform which will use the provided content type hint
      * to make the file parser detection more efficient.
      */
-    public Read withContentTypeHint(String contentType) {
+    public ParseAll withContentTypeHint(String contentType) {
       checkNotNull(contentType, "ContentType cannot be empty.");
       Metadata metadata = new Metadata();
       metadata.add(Metadata.CONTENT_TYPE, contentType);
@@ -160,7 +203,7 @@ public class TikaIO {
      * Returns a new transform which will use the provided input metadata
      * for parsing the files.
      */
-    public Read withInputMetadata(Metadata metadata) {
+    public ParseAll withInputMetadata(Metadata metadata) {
       Metadata inputMetadata = this.getInputMetadata();
       if (inputMetadata != null) {
         for (String name : metadata.names()) {
@@ -172,88 +215,15 @@ public class TikaIO {
       return toBuilder().setInputMetadata(inputMetadata).build();
     }
 
-    /**
-     * Returns a new transform which will report the metadata.
-     */
-    public Read withReadOutputMetadata(Boolean value) {
-      return toBuilder().setReadOutputMetadata(value).build();
-    }
-
-    /**
-     * Returns a new transform which will use the specified queue poll time.
-     */
-    public Read withQueuePollTime(Long value) {
-      return toBuilder().setQueuePollTime(value).build();
-    }
-
-    /**
-     * Returns a new transform which will use the specified queue max poll time.
-     */
-    public Read withQueueMaxPollTime(Long value) {
-      return toBuilder().setQueueMaxPollTime(value).build();
-    }
-
-    /**
-     * Returns a new transform which will operate on the text blocks with the
-     * given minimum text length.
-     */
-    public Read withMinimumTextlength(Integer value) {
-      return toBuilder().setMinimumTextLength(value).build();
-    }
-
-    /**
-     * Returns a new transform which will use the synchronous reader.
-     */
-    public Read withParseSynchronously(Boolean value) {
-      return toBuilder().setParseSynchronously(value).build();
-    }
-
-    /**
-     * Path to Tika configuration resource.
-     */
-    public Read withOptions(TikaOptions options) {
-      checkNotNull(options, "TikaOptions cannot be empty.");
-      Builder builder = toBuilder();
-      builder.setFilepattern(StaticValueProvider.of(options.getInput()))
-             .setQueuePollTime(options.getQueuePollTime())
-             .setQueueMaxPollTime(options.getQueueMaxPollTime())
-             .setMinimumTextLength(options.getMinimumTextLength())
-             .setParseSynchronously(options.getParseSynchronously());
-      if (options.getContentTypeHint() != null) {
-        Metadata metadata = this.getInputMetadata();
-        if (metadata == null) {
-            metadata = new Metadata();
-        }
-        metadata.add(Metadata.CONTENT_TYPE, options.getContentTypeHint());
-        builder.setInputMetadata(metadata);
-      }
-      if (options.getTikaConfigPath() != null) {
-        builder.setTikaConfigPath(StaticValueProvider.of(options.getTikaConfigPath()));
-      }
-      if (Boolean.TRUE.equals(options.getReadOutputMetadata())) {
-        builder.setReadOutputMetadata(options.getReadOutputMetadata());
-      }
-      return builder.build();
-    }
-
     @Override
-    public PCollection<String> expand(PBegin input) {
-      checkNotNull(this.getFilepattern(), "Filepattern cannot be empty.");
-      final Bounded<String> read = org.apache.beam.sdk.io.Read.from(new TikaSource(this));
-      PCollection<String> pcol = input.getPipeline().apply(read);
-      pcol.setCoder(getDefaultOutputCoder());
-      return pcol;
+    public PCollection<ParseResult> expand(PCollection<ReadableFile> input) {
+      return input.apply(ParDo.of(new ParseToStringFn(this)));
     }
 
     @Override
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
 
-      String filepatternDisplay = getFilepattern().isAccessible()
-        ? getFilepattern().get() : getFilepattern().toString();
-      builder
-          .addIfNotNull(DisplayData.item("filePattern", filepatternDisplay)
-            .withLabel("File Pattern"));
       if (getTikaConfigPath() != null) {
         String tikaConfigPathDisplay = getTikaConfigPath().isAccessible()
           ? getTikaConfigPath().get() : getTikaConfigPath().toString();
@@ -262,49 +232,55 @@ public class TikaIO {
       }
       Metadata metadata = getInputMetadata();
       if (metadata != null) {
-        StringBuilder sb = new StringBuilder();
-        sb.append('[');
-        for (String name : metadata.names()) {
-            if (sb.length() > 1) {
-              sb.append(',');
-            }
-            sb.append(name).append('=').append(metadata.get(name));
-        }
-        sb.append(']');
+        //TODO: use metadata.toString() only without a trim() once Apache Tika 1.17 gets released
         builder
-            .add(DisplayData.item("inputMetadata", sb.toString())
+            .add(DisplayData.item("inputMetadata", metadata.toString().trim())
             .withLabel("Input Metadata"));
       }
-      if (Boolean.TRUE.equals(getParseSynchronously())) {
-        builder
-          .add(DisplayData.item("parseMode", "synchronous")
-            .withLabel("Parse Mode"));
-      } else {
-        builder
-          .add(DisplayData.item("parseMode", "asynchronous")
-            .withLabel("Parse Mode"));
-        builder
-          .add(DisplayData.item("queuePollTime", getQueuePollTime().toString())
-            .withLabel("Queue Poll Time"))
-        .add(DisplayData.item("queueMaxPollTime", getQueueMaxPollTime().toString())
-          .withLabel("Queue Max Poll Time"));
-      }
-      Integer minTextLen = getMinimumTextLength();
-      if (minTextLen != null && minTextLen > 0) {
-        builder
-        .add(DisplayData.item("minTextLen", getMinimumTextLength().toString())
-          .withLabel("Minimum Text Length"));
+    }
+
+    private static class ParseToStringFn extends DoFn<ReadableFile, ParseResult> {
+
+      private static final long serialVersionUID = 6837207505313720989L;
+      private final TikaIO.ParseAll spec;
+      private TikaConfig tikaConfig;
+
+      ParseToStringFn(TikaIO.ParseAll spec) {
+        this.spec = spec;
       }
-      if (Boolean.TRUE.equals(getReadOutputMetadata())) {
-        builder
-          .add(DisplayData.item("readOutputMetadata", "true")
-            .withLabel("Read Output Metadata"));
+
+      @Setup
+      public void setup() throws Exception {
+        if (spec.getTikaConfigPath() != null) {
+          ResourceId configResource =
+              FileSystems.matchSingleFileSpec(spec.getTikaConfigPath().get()).resourceId();
+          tikaConfig = new TikaConfig(
+                           Channels.newInputStream(FileSystems.open(configResource)));
+        }
       }
-    }
 
-    @Override
-    protected Coder<String> getDefaultOutputCoder() {
-      return StringUtf8Coder.of();
+      @ProcessElement
+      public void processElement(ProcessContext c) throws Exception {
+        ReadableFile file = c.element();
+        InputStream stream = Channels.newInputStream(file.open());
+        try (InputStream tikaStream = TikaInputStream.get(stream)) {
+
+          final Parser parser = tikaConfig == null
+              ? new AutoDetectParser() : new AutoDetectParser(tikaConfig);
+
+          final ParseContext context = new ParseContext();
+          context.set(Parser.class, parser);
+          Metadata tikaMetadata = spec.getInputMetadata() != null
+            ? spec.getInputMetadata() : new org.apache.tika.metadata.Metadata();
+
+          ContentHandler tikaHandler = new ToTextContentHandler();
+          parser.parse(tikaStream, tikaHandler, tikaMetadata, context);
+
+          c.output(new ParseResult(file.getMetadata().resourceId().toString(),
+              tikaHandler.toString(),
+              tikaMetadata));
+        }
+      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java
deleted file mode 100644
index fb97678..0000000
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaOptions.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import org.apache.beam.sdk.options.Default;
-import org.apache.beam.sdk.options.Description;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.beam.sdk.options.Validation;
-
-/**
- * TikaInput Options to support the command-line applications.
- */
-public interface TikaOptions extends PipelineOptions {
-
-  @Description("Input path")
-  @Validation.Required
-  String getInput();
-  void setInput(String value);
-
-  @Description("Tika Config path")
-  String getTikaConfigPath();
-  void setTikaConfigPath(String value);
-
-  @Description("Tika Parser Content Type hint")
-  String getContentTypeHint();
-  void setContentTypeHint(String value);
-
-  @Description("Metadata report status")
-  @Default.Boolean(false)
-  Boolean getReadOutputMetadata();
-  void setReadOutputMetadata(Boolean value);
-
-  @Description("Optional use of the synchronous reader")
-  @Default.Boolean(false)
-  Boolean getParseSynchronously();
-  void setParseSynchronously(Boolean value);
-
-  @Description("Tika Parser queue poll time in milliseconds")
-  @Default.Long(TikaIO.Read.DEFAULT_QUEUE_POLL_TIME)
-  Long getQueuePollTime();
-  void setQueuePollTime(Long value);
-
-  @Description("Tika Parser queue maximum poll time in milliseconds")
-  @Default.Long(TikaIO.Read.DEFAULT_QUEUE_MAX_POLL_TIME)
-  Long getQueueMaxPollTime();
-  void setQueueMaxPollTime(Long value);
-
-  @Description("Minumin text fragment length for Tika Parser to report")
-  @Default.Integer(0)
-  Integer getMinimumTextLength();
-  void setMinimumTextLength(Integer value);
-
-  @Description("Pipeline name")
-  @Default.String("TikaRead")
-  String getPipelineName();
-  void setPipelineName(String value);
-
-  @Description("Output path")
-  @Default.String("/tmp/tika/out")
-  String getOutput();
-  void setOutput(String value);
-
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java
deleted file mode 100644
index 7c8852b..0000000
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaSource.java
+++ /dev/null
@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.NoSuchElementException;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import javax.annotation.Nullable;
-
-import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.FileSystems;
-import org.apache.beam.sdk.io.Source;
-import org.apache.beam.sdk.io.fs.MatchResult;
-import org.apache.beam.sdk.io.fs.MatchResult.Metadata;
-import org.apache.beam.sdk.options.PipelineOptions;
-import org.apache.tika.config.TikaConfig;
-import org.apache.tika.exception.TikaException;
-import org.apache.tika.io.TikaInputStream;
-import org.apache.tika.parser.AutoDetectParser;
-import org.apache.tika.parser.ParseContext;
-import org.apache.tika.parser.Parser;
-import org.joda.time.Instant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-import org.xml.sax.helpers.DefaultHandler;
-
-/**
- * Implementation detail of {@link TikaIO.Read}.
- *
- * <p>A {@link Source} which can represent the content of the files parsed by Apache Tika.
- */
-class TikaSource extends BoundedSource<String> {
-  private static final long serialVersionUID = -509574062910491122L;
-  private static final Logger LOG = LoggerFactory.getLogger(TikaSource.class);
-
-  @Nullable
-  private MatchResult.Metadata singleFileMetadata;
-  private final Mode mode;
-  private final TikaIO.Read spec;
-
-  /**
-   * Source mode.
-   */
-  public enum Mode {
-    FILEPATTERN, SINGLE_FILE
-  }
-
-  TikaSource(TikaIO.Read spec) {
-    this.mode = Mode.FILEPATTERN;
-    this.spec = spec;
-  }
-
-  TikaSource(Metadata fileMetadata, TikaIO.Read spec) {
-    mode = Mode.SINGLE_FILE;
-    this.singleFileMetadata = checkNotNull(fileMetadata, "fileMetadata");
-    this.spec = spec;
-  }
-
-  @Override
-  public BoundedReader<String> createReader(PipelineOptions options) throws IOException {
-    this.validate();
-    checkState(spec.getFilepattern().isAccessible(),
-        "Cannot create a Tika reader without access to the file"
-        + " or pattern specification: {}.", spec.getFilepattern());
-    if (spec.getTikaConfigPath() != null) {
-      checkState(spec.getTikaConfigPath().isAccessible(),
-        "Cannot create a Tika reader without access to its configuration",
-        spec.getTikaConfigPath());
-    }
-
-    String fileOrPattern = spec.getFilepattern().get();
-    if (mode == Mode.FILEPATTERN) {
-      List<Metadata> fileMetadata = expandFilePattern(fileOrPattern);
-      List<TikaReader> fileReaders = new ArrayList<>();
-      for (Metadata metadata : fileMetadata) {
-        fileReaders.add(new TikaReader(this, metadata.resourceId().toString()));
-      }
-      if (fileReaders.size() == 1) {
-        return fileReaders.get(0);
-      }
-      return new FilePatternTikaReader(this, fileReaders);
-    } else {
-      return new TikaReader(this, singleFileMetadata.resourceId().toString());
-    }
-
-  }
-
-  @Override
-  public List<? extends TikaSource> split(long desiredBundleSizeBytes, PipelineOptions options)
-    throws Exception {
-    if (mode == Mode.SINGLE_FILE) {
-      return ImmutableList.of(this);
-    } else {
-      List<Metadata> fileMetadata = expandFilePattern(spec.getFilepattern().get());
-
-      List<TikaSource> splitResults = new LinkedList<>();
-      for (Metadata metadata : fileMetadata) {
-        splitResults.add(new TikaSource(metadata, spec));
-      }
-      return splitResults;
-    }
-  }
-
-  public TikaIO.Read getTikaInputRead() {
-    return spec;
-  }
-
-  @Override
-  public Coder<String> getDefaultOutputCoder() {
-    return StringUtf8Coder.of();
-  }
-
-  @Override
-  public void validate() {
-    switch (mode) {
-    case FILEPATTERN:
-      checkArgument(this.singleFileMetadata == null,
-        "Unexpected initialized singleFileMetadata value");
-      break;
-    case SINGLE_FILE:
-      checkNotNull(this.singleFileMetadata,
-        "Unexpected null singleFileMetadata value");
-      break;
-    default:
-      throw new IllegalStateException("Unknown mode: " + mode);
-    }
-  }
-
-  @Override
-  public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
-    long totalSize = 0;
-    List<Metadata> fileMetadata = expandFilePattern(spec.getFilepattern().get());
-    for (Metadata metadata : fileMetadata) {
-      totalSize += metadata.sizeBytes();
-    }
-    return totalSize;
-  }
-
-  Mode getMode() {
-    return this.mode;
-  }
-
-  Metadata getSingleFileMetadata() {
-    return this.singleFileMetadata;
-  }
-
-  private static List<Metadata> expandFilePattern(String fileOrPattern) throws IOException {
-    MatchResult matches = Iterables.getOnlyElement(
-      FileSystems.match(Collections.singletonList(fileOrPattern)));
-    LOG.info("Matched {} files for pattern {}", matches.metadata().size(), fileOrPattern);
-    List<Metadata> metadata = ImmutableList.copyOf(matches.metadata());
-    checkArgument(!metadata.isEmpty(),
-      "Unable to find any files matching %s", fileOrPattern);
-
-    return metadata;
-  }
-
-  /**
-   *  FilePatternTikaReader.
-   *  TODO: This is mostly a copy of FileBasedSource internal file-pattern reader
-   *        so that code would need to be generalized as part of the future contribution
-   */
-  static class FilePatternTikaReader extends BoundedReader<String> {
-    private final TikaSource source;
-    final ListIterator<TikaReader> fileReadersIterator;
-    TikaReader currentReader = null;
-
-    public FilePatternTikaReader(TikaSource source, List<TikaReader> fileReaders) {
-      this.source = source;
-      this.fileReadersIterator = fileReaders.listIterator();
-    }
-
-    @Override
-    public boolean start() throws IOException {
-      return startNextNonemptyReader();
-    }
-
-    @Override
-    public boolean advance() throws IOException {
-      checkState(currentReader != null, "Call start() before advance()");
-      if (currentReader.advance()) {
-        return true;
-      }
-      return startNextNonemptyReader();
-    }
-
-    private boolean startNextNonemptyReader() throws IOException {
-      while (fileReadersIterator.hasNext()) {
-        currentReader = fileReadersIterator.next();
-        if (currentReader.start()) {
-          return true;
-        }
-        currentReader.close();
-      }
-      return false;
-    }
-
-    @Override
-    public String getCurrent() throws NoSuchElementException {
-      return currentReader.getCurrent();
-    }
-
-    @Override
-    public Instant getCurrentTimestamp() throws NoSuchElementException {
-      return currentReader.getCurrentTimestamp();
-    }
-
-    @Override
-    public void close() throws IOException {
-      if (currentReader != null) {
-        currentReader.close();
-      }
-      while (fileReadersIterator.hasNext()) {
-        fileReadersIterator.next().close();
-      }
-    }
-
-    @Override
-    public TikaSource getCurrentSource() {
-      return source;
-    }
-  }
-
-  static class TikaReader extends BoundedReader<String> {
-    private ExecutorService execService;
-    private final ContentHandlerImpl tikaHandler = new ContentHandlerImpl();
-    private String current;
-    private TikaSource source;
-    private String filePath;
-    private TikaIO.Read spec;
-    private org.apache.tika.metadata.Metadata tikaMetadata;
-    private Iterator<String> metadataIterator;
-
-    TikaReader(TikaSource source, String filePath) {
-      this.source = source;
-      this.filePath = filePath;
-      this.spec = source.getTikaInputRead();
-    }
-
-    @Override
-    public boolean start() throws IOException {
-      final InputStream is = TikaInputStream.get(Paths.get(filePath));
-      TikaConfig tikaConfig = null;
-      if (spec.getTikaConfigPath() != null) {
-        try {
-          tikaConfig = new TikaConfig(spec.getTikaConfigPath().get());
-        } catch (TikaException | SAXException e) {
-          throw new IOException(e);
-        }
-      }
-      final Parser parser = tikaConfig == null ? new AutoDetectParser()
-          : new AutoDetectParser(tikaConfig);
-      final ParseContext context = new ParseContext();
-      context.set(Parser.class, parser);
-      tikaMetadata = spec.getInputMetadata() != null ? spec.getInputMetadata()
-          : new org.apache.tika.metadata.Metadata();
-
-      if (spec.getMinimumTextLength() != null) {
-        tikaHandler.setMinTextLength(spec.getMinimumTextLength());
-      }
-
-      if (!Boolean.TRUE.equals(spec.getParseSynchronously())) {
-        // Try to parse the file on the executor thread to make the best effort
-        // at letting the pipeline thread advancing over the file content
-        // without immediately parsing all of it
-        execService = Executors.newFixedThreadPool(1);
-        execService.submit(new Runnable() {
-          public void run() {
-            try {
-              parser.parse(is, tikaHandler, tikaMetadata, context);
-              is.close();
-            } catch (Exception ex) {
-              tikaHandler.setParseException(ex);
-            }
-          }
-        });
-      } else {
-        // Some parsers might not be able to report the content in chunks.
-        // It does not make sense to create extra threads in such cases
-        try {
-          parser.parse(is, tikaHandler, tikaMetadata, context);
-        } catch (Exception ex) {
-          throw new IOException(ex);
-        } finally {
-          is.close();
-        }
-      }
-      return advanceToNext();
-    }
-
-    @Override
-    public boolean advance() throws IOException {
-      checkState(current != null, "Call start() before advance()");
-      return advanceToNext();
-    }
-
-    protected boolean advanceToNext() throws IOException {
-      current = null;
-      // The content is reported first
-      if (metadataIterator == null) {
-        // Check if some content is already available
-        current = tikaHandler.getCurrent();
-
-        if (current == null && !Boolean.TRUE.equals(spec.getParseSynchronously())) {
-          long maxPollTime = 0;
-          long configuredMaxPollTime = spec.getQueueMaxPollTime() == null
-              ? TikaIO.Read.DEFAULT_QUEUE_MAX_POLL_TIME : spec.getQueueMaxPollTime();
-          long configuredPollTime = spec.getQueuePollTime() == null
-              ? TikaIO.Read.DEFAULT_QUEUE_POLL_TIME : spec.getQueuePollTime();
-
-          // Poll the queue till the next piece of data is available
-          while (current == null && maxPollTime < configuredMaxPollTime) {
-            boolean docEnded = tikaHandler.waitForNext(configuredPollTime);
-            current = tikaHandler.getCurrent();
-            // End of Document ?
-            if (docEnded) {
-              break;
-            }
-            maxPollTime += spec.getQueuePollTime();
-          }
-        }
-        // No more content ?
-        if (current == null && Boolean.TRUE.equals(spec.getReadOutputMetadata())) {
-          // Time to report the metadata
-          metadataIterator = Arrays.asList(tikaMetadata.names()).iterator();
-        }
-      }
-
-      if (metadataIterator != null && metadataIterator.hasNext()) {
-          String key = metadataIterator.next();
-          // The metadata name/value separator can be configured if needed
-          current = key + "=" + tikaMetadata.get(key);
-      }
-      return current != null;
-    }
-
-    @Override
-    public String getCurrent() throws NoSuchElementException {
-      if (current == null) {
-        throw new NoSuchElementException();
-      }
-      return current;
-    }
-
-    @Override
-    public void close() throws IOException {
-      if (execService != null) {
-          execService.shutdown();
-      }
-    }
-
-    ExecutorService getExecutorService() {
-      return execService;
-    }
-
-    @Override
-    public BoundedSource<String> getCurrentSource() {
-      return source;
-    }
-  }
-
-  /**
-   * Tika Parser Content Handler.
-   */
-  static class ContentHandlerImpl extends DefaultHandler {
-    private Queue<String> queue = new ConcurrentLinkedQueue<>();
-    private volatile boolean documentEnded;
-    private volatile Exception parseException;
-    private volatile String current;
-    private int minTextLength;
-
-    @Override
-    public void characters(char ch[], int start, int length) throws SAXException {
-      String value = new String(ch, start, length).trim();
-      if (!value.isEmpty()) {
-        if (minTextLength <= 0) {
-          queue.add(value);
-        } else {
-          current = current == null ? value : current + " " + value;
-          if (current.length() >= minTextLength) {
-            queue.add(current);
-            current = null;
-          }
-        }
-      }
-    }
-
-    public void setParseException(Exception ex) {
-      this.parseException = ex;
-    }
-
-    public synchronized boolean waitForNext(long pollTime) throws IOException {
-      if (!documentEnded) {
-        try {
-          wait(pollTime);
-        } catch (InterruptedException ex) {
-          // continue;
-        }
-      }
-      return documentEnded;
-    }
-
-    @Override
-    public synchronized void endDocument() throws SAXException {
-      this.documentEnded = true;
-      notify();
-    }
-
-    public String getCurrent() throws IOException {
-      checkParseException();
-      String value = queue.poll();
-      if (value == null && documentEnded) {
-        return current;
-      } else {
-        return value;
-      }
-    }
-    public void checkParseException() throws IOException {
-      if (parseException != null) {
-        throw new IOException(parseException);
-      }
-    }
-
-    public void setMinTextLength(int minTextLength) {
-      this.minTextLength = minTextLength;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
new file mode 100644
index 0000000..fd86152
--- /dev/null
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.tika;
+
+import static org.junit.Assert.assertEquals;
+import org.apache.tika.metadata.Metadata;
+import org.junit.Test;
+
+/**
+ * Tests ParseResult.
+ */
+public class ParseResultTest {
+  @Test
+  public void testEqualsAndHashCode() {
+    ParseResult p1 = new ParseResult("a.txt", "hello", getMetadata());
+    ParseResult p2 = new ParseResult("a.txt", "hello", getMetadata());
+    assertEquals(p1, p2);
+    assertEquals(p1.hashCode(), p2.hashCode());
+  }
+
+  static Metadata getMetadata() {
+    Metadata m = new Metadata();
+    m.add("Author", "BeamTikaUser");
+    m.add("Author", "BeamTikaUser2");
+    m.add("Date", "2017-09-01");
+    return m;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
index 40ff569..a985b0a 100644
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
@@ -18,14 +18,14 @@
 package org.apache.beam.sdk.io.tika;
 
 import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
+import static org.hamcrest.Matchers.isA;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import java.io.IOException;
 
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.io.Compression;
+import org.apache.beam.sdk.io.FileIO;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -33,231 +33,171 @@ import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.tika.exception.TikaException;
-import org.junit.Ignore;
+import org.apache.tika.metadata.Metadata;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 /**
  * Tests TikaInput.
  */
 public class TikaIOTest {
-  private static final String[] PDF_FILE = new String[] {
-      "Combining", "can help to ingest", "Apache Beam", "in most known formats.",
-      "the content from the files", "and", "Apache Tika"
-  };
-  private static final String[] PDF_ZIP_FILE = new String[] {
-      "Combining", "can help to ingest", "Apache Beam", "in most known formats.",
-      "the content from the files", "and", "Apache Tika",
-      "apache-beam-tika.pdf"
-  };
-  private static final String[] ODT_FILE = new String[] {
-      "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
-      "the content from the files", "and", "Apache Tika"
-  };
-  private static final String[] ODT_FILE_WITH_METADATA = new String[] {
-      "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
-      "the content from the files", "and", "Apache Tika",
-      "Author=BeamTikaUser"
-  };
-  private static final String[] ODT_FILE_WITH_MIN_TEXT_LEN = new String[] {
-      "Combining Apache Beam", "and Apache Tika can help to ingest", "in most known formats.",
-      "the content from the files"
-  };
-  private static final String[] ODT_FILES = new String[] {
-      "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
-      "the content from the files", "and", "Apache Tika",
-      "Open Office", "Text", "PDF", "Excel", "Scientific",
-      "and other formats", "are supported."
-  };
+  private static final String PDF_FILE =
+      "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+      + "Combining\n\nApache Beam\n\nand\n\nApache Tika\n\ncan help to ingest\n\n"
+      + "the content from the files\n\nin most known formats.\n\n\n";
+
+  private static final String PDF_ZIP_FILE =
+      "\n\n\n\n\n\n\n\napache-beam-tika.pdf\n\n\nCombining\n\n\nApache Beam\n\n\n"
+      + "and\n\n\nApache Tika\n\n\ncan help to ingest\n\n\nthe content from the files\n\n\n"
+      + "in most known formats.\n\n\n\n\n\n\n";
+
+  private static final String ODT_FILE =
+      "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+      + "Combining\nApache Beam\nand\nApache Tika\ncan help to ingest\nthe content from the"
+      + " files\nin most known formats.\n";
+
+  private static final String ODT_FILE2 =
+      "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+      + "Open Office\nPDF\nExcel\nText\nScientific\nand other formats\nare supported.\n";
 
   @Rule
   public TestPipeline p = TestPipeline.create();
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
 
-  @Ignore
   @Test
-  public void testReadPdfFile() throws IOException {
+  public void testParsePdfFile() throws IOException {
 
     String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
 
-    doTestReadFiles(resourcePath, PDF_FILE);
+    doTestParse(resourcePath, new ParseResult(resourcePath, PDF_FILE));
   }
 
-  @Test
-  public void testReadZipPdfFile() throws IOException {
-
-    String resourcePath = getClass().getResource("/apache-beam-tika-pdf.zip").getPath();
-
-    doTestReadFiles(resourcePath, PDF_ZIP_FILE);
+  private void doTestParse(String resourcePath, ParseResult... expectedResults)
+      throws IOException {
+     PCollection<ParseResult> output =
+         p.apply("ParseAll", TikaIO.parse().filepattern(resourcePath))
+         .apply(ParDo.of(new FilterMetadataFn()));
+     PAssert.that(output).containsInAnyOrder(expectedResults);
+     p.run();
   }
 
   @Test
-  public void testReadOdtFile() throws IOException {
+  public void testParseAllPdfFile() throws IOException {
 
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
+    String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
 
-    doTestReadFiles(resourcePath, ODT_FILE);
+    doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_FILE));
   }
 
   @Test
-  public void testReadOdtFiles() throws IOException {
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-    resourcePath = resourcePath.replace("apache-beam-tika1", "*");
+  public void testParseAllZipPdfFile() throws IOException {
 
-    doTestReadFiles(resourcePath, ODT_FILES);
-  }
+    String resourcePath = getClass().getResource("/apache-beam-tika-pdf.zip").getPath();
 
-  private void doTestReadFiles(String resourcePath, String[] expected) throws IOException {
-    PCollection<String> output = p.apply("ParseFiles", TikaIO.read().from(resourcePath));
-    PAssert.that(output).containsInAnyOrder(expected);
-    p.run();
+    doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_ZIP_FILE));
   }
 
   @Test
-  public void testReadOdtFileWithMetadata() throws IOException {
+  public void testParseAllOdtFile() throws IOException {
 
     String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
 
-    PCollection<String> output = p.apply("ParseOdtFile",
-        TikaIO.read().from(resourcePath).withReadOutputMetadata(true))
-        .apply(ParDo.of(new FilterMetadataFn()));
-    PAssert.that(output).containsInAnyOrder(ODT_FILE_WITH_METADATA);
-    p.run();
+    doTestParseAll(resourcePath, new ParseResult(resourcePath, ODT_FILE, getOdtMetadata()));
   }
 
   @Test
-  public void testReadOdtFileWithMinTextLength() throws IOException {
+  public void testParseAllOdtFiles() throws IOException {
+    String resourcePath1 = getClass().getResource("/apache-beam-tika1.odt").getPath();
+    String resourcePath2 = getClass().getResource("/apache-beam-tika2.odt").getPath();
+    String resourcePath = resourcePath1.replace("apache-beam-tika1", "*");
 
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-
-    PCollection<String> output = p.apply("ParseOdtFile",
-        TikaIO.read().from(resourcePath).withMinimumTextlength(20));
-    PAssert.that(output).containsInAnyOrder(ODT_FILE_WITH_MIN_TEXT_LEN);
-    p.run();
+    doTestParseAll(resourcePath, new ParseResult(resourcePath1, ODT_FILE, getOdtMetadata()),
+        new ParseResult(resourcePath2, ODT_FILE2));
   }
 
-  @Test
-  public void testReadPdfFileSync() throws IOException {
-
-    String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
-
-    PCollection<String> output = p.apply("ParsePdfFile",
-        TikaIO.read().from(resourcePath).withParseSynchronously(true));
-    PAssert.that(output).containsInAnyOrder(PDF_FILE);
+  private void doTestParseAll(String resourcePath, ParseResult... expectedResults)
+     throws IOException {
+    PCollection<ParseResult> output =
+        p.apply("ParseFiles", FileIO.match().filepattern(resourcePath))
+        .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+        .apply(TikaIO.parseAll())
+        .apply(ParDo.of(new FilterMetadataFn()));
+    PAssert.that(output).containsInAnyOrder(expectedResults);
     p.run();
   }
 
   @Test
-  public void testReadDamagedPdfFile() throws IOException {
+  public void testParseAllDamagedPdfFile() throws IOException {
+    thrown.expectCause(isA(TikaException.class));
+    String resourcePath = getClass().getResource("/damaged.pdf").getPath();
 
-    doTestReadDamagedPdfFile(false);
+    p.apply("ParseInvalidPdfFile", FileIO.match().filepattern(resourcePath))
+      .apply(FileIO.readMatches())
+      .apply(TikaIO.parseAll());
+    p.run();
   }
 
   @Test
-  public void testReadDamagedPdfFileSync() throws IOException {
-    doTestReadDamagedPdfFile(true);
-  }
+  public void testParseDisplayData() {
+    TikaIO.Parse parse = TikaIO.parse().filepattern("file.pdf");
 
-  private void doTestReadDamagedPdfFile(boolean sync) throws IOException {
+    DisplayData displayData = DisplayData.from(parse);
 
-    String resourcePath = getClass().getResource("/damaged.pdf").getPath();
-
-    p.apply("ParseInvalidPdfFile",
-        TikaIO.read().from(resourcePath).withParseSynchronously(sync));
-    try {
-        p.run();
-        fail("Transform failure is expected");
-    } catch (RuntimeException ex) {
-      assertTrue(ex.getCause().getCause() instanceof TikaException);
-    }
+    assertThat(displayData, hasDisplayItem("filePattern", "file.pdf"));
+    assertEquals(1, displayData.items().size());
   }
 
   @Test
-  public void testReadDisplayData() {
-    TikaIO.Read read = TikaIO.read()
-        .from("foo.*")
+  public void testParseAllDisplayData() {
+    TikaIO.ParseAll parseAll = TikaIO.parseAll()
         .withTikaConfigPath("tikaconfigpath")
-        .withContentTypeHint("application/pdf")
-        .withMinimumTextlength(100)
-        .withReadOutputMetadata(true);
+        .withContentTypeHint("application/pdf");
 
-    DisplayData displayData = DisplayData.from(read);
+    DisplayData displayData = DisplayData.from(parseAll);
 
-    assertThat(displayData, hasDisplayItem("filePattern", "foo.*"));
     assertThat(displayData, hasDisplayItem("tikaConfigPath", "tikaconfigpath"));
     assertThat(displayData, hasDisplayItem("inputMetadata",
-        "[Content-Type=application/pdf]"));
-    assertThat(displayData, hasDisplayItem("readOutputMetadata", "true"));
-    assertThat(displayData, hasDisplayItem("parseMode", "asynchronous"));
-    assertThat(displayData, hasDisplayItem("queuePollTime", "50"));
-    assertThat(displayData, hasDisplayItem("queueMaxPollTime", "3000"));
-    assertThat(displayData, hasDisplayItem("minTextLen", "100"));
-    assertEquals(8, displayData.items().size());
-  }
-
-  @Test
-  public void testReadDisplayDataSyncMode() {
-    TikaIO.Read read = TikaIO.read()
-        .from("foo.*")
-        .withParseSynchronously(true);
-
-    DisplayData displayData = DisplayData.from(read);
-
-    assertThat(displayData, hasDisplayItem("filePattern", "foo.*"));
-    assertThat(displayData, hasDisplayItem("parseMode", "synchronous"));
+        "Content-Type=application/pdf"));
     assertEquals(2, displayData.items().size());
   }
 
   @Test
-  public void testReadDisplayDataWithDefaultOptions() {
-    final String[] args = new String[]{"--input=/input/tika.pdf"};
-    TikaIO.Read read = TikaIO.read().withOptions(createOptions(args));
+  public void testParseAllDisplayDataWithCustomOptions() {
+    TikaIO.ParseAll parseAll = TikaIO.parseAll()
+        .withTikaConfigPath("/tikaConfigPath")
+        .withContentTypeHint("application/pdf");
 
-    DisplayData displayData = DisplayData.from(read);
+    DisplayData displayData = DisplayData.from(parseAll);
 
-    assertThat(displayData, hasDisplayItem("filePattern", "/input/tika.pdf"));
-    assertThat(displayData, hasDisplayItem("parseMode", "asynchronous"));
-    assertThat(displayData, hasDisplayItem("queuePollTime", "50"));
-    assertThat(displayData, hasDisplayItem("queueMaxPollTime", "3000"));
-    assertEquals(4, displayData.items().size());
-  }
-  @Test
-  public void testReadDisplayDataWithCustomOptions() {
-    final String[] args = new String[]{"--input=/input/tika.pdf",
-                                       "--tikaConfigPath=/tikaConfigPath",
-                                       "--queuePollTime=10",
-                                       "--queueMaxPollTime=1000",
-                                       "--contentTypeHint=application/pdf",
-                                       "--readOutputMetadata=true"};
-    TikaIO.Read read = TikaIO.read().withOptions(createOptions(args));
-
-    DisplayData displayData = DisplayData.from(read);
-
-    assertThat(displayData, hasDisplayItem("filePattern", "/input/tika.pdf"));
     assertThat(displayData, hasDisplayItem("tikaConfigPath", "/tikaConfigPath"));
-    assertThat(displayData, hasDisplayItem("parseMode", "asynchronous"));
-    assertThat(displayData, hasDisplayItem("queuePollTime", "10"));
-    assertThat(displayData, hasDisplayItem("queueMaxPollTime", "1000"));
     assertThat(displayData, hasDisplayItem("inputMetadata",
-        "[Content-Type=application/pdf]"));
-    assertThat(displayData, hasDisplayItem("readOutputMetadata", "true"));
-    assertEquals(7, displayData.items().size());
-  }
-
-  private static TikaOptions createOptions(String[] args) {
-    return PipelineOptionsFactory.fromArgs(args)
-        .withValidation().as(TikaOptions.class);
+        "Content-Type=application/pdf"));
+    assertEquals(2, displayData.items().size());
   }
 
-  static class FilterMetadataFn extends DoFn<String, String> {
+  static class FilterMetadataFn extends DoFn<ParseResult, ParseResult> {
     private static final long serialVersionUID = 6338014219600516621L;
 
     @ProcessElement
     public void processElement(ProcessContext c) {
-      String word = c.element();
-      if (word.contains("=") && !word.startsWith("Author")) {
-        return;
+      ParseResult result = c.element();
+      Metadata m = new Metadata();
+      // Files contain many metadata properties. This function drops all but the "Author"
+      // property manually added to "apache-beam-tika1.odt" resource only to make
+      // the tests simpler
+      if (result.getFileLocation().endsWith("apache-beam-tika1.odt")) {
+          m.set("Author", result.getMetadata().get("Author"));
       }
-      c.output(word);
+      ParseResult newResult = new ParseResult(result.getFileLocation(), result.getContent(), m);
+      c.output(newResult);
     }
   }
+
+  static Metadata getOdtMetadata() {
+    Metadata m = new Metadata();
+    m.set("Author", "BeamTikaUser");
+    return m;
+  }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java
deleted file mode 100644
index 5c4e754..0000000
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaReaderTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.beam.sdk.io.tika.TikaSource.TikaReader;
-import org.junit.Test;
-
-/**
- * Tests TikaReader.
- */
-public class TikaReaderTest {
-  private static final List<String> ODT_FILE = Arrays.asList(
-      "Combining", "can help to ingest", "Apache", "Beam", "in most known formats.",
-      "the content from the files", "and", "Apache Tika");
-
-  @Test
-  public void testOdtFileAsyncReader() throws Exception {
-    doTestOdtFileReader(false);
-  }
-  @Test
-  public void testOdtFileSyncReader() throws Exception {
-    doTestOdtFileReader(true);
-  }
-  private void doTestOdtFileReader(boolean sync) throws Exception {
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-    TikaSource source = new TikaSource(TikaIO.read()
-                                                .withParseSynchronously(sync)
-                                                .from(resourcePath));
-    TikaReader reader = (TikaReader) source.createReader(null);
-
-    List<String> content = new LinkedList<String>();
-    for (boolean available = reader.start(); available; available = reader.advance()) {
-      content.add(reader.getCurrent());
-    }
-    assertTrue(content.containsAll(ODT_FILE));
-    if (!sync) {
-      assertNotNull(reader.getExecutorService());
-    } else {
-      assertNull(reader.getExecutorService());
-    }
-    reader.close();
-  }
-
-  @Test
-  public void testOdtFilesReader() throws Exception {
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-    String filePattern = resourcePath.replace("apache-beam-tika1", "*");
-
-    TikaSource source = new TikaSource(TikaIO.read().from(filePattern));
-    TikaSource.FilePatternTikaReader reader =
-        (TikaSource.FilePatternTikaReader) source.createReader(null);
-    List<String> content = new LinkedList<String>();
-    for (boolean available = reader.start(); available; available = reader.advance()) {
-      content.add(reader.getCurrent());
-    }
-    assertTrue(content.containsAll(ODT_FILE));
-    reader.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/beam/blob/ba93dd39/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java
deleted file mode 100644
index 550f469..0000000
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaSourceTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.sdk.io.tika;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-
-import org.apache.beam.sdk.coders.StringUtf8Coder;
-import org.apache.beam.sdk.io.tika.TikaSource.TikaReader;
-import org.junit.Test;
-
-/**
- * Tests TikaSource.
- */
-public class TikaSourceTest {
-
-  @Test
-  public void testOdtFileSource() throws Exception {
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-    TikaSource source = new TikaSource(TikaIO.read().from(resourcePath));
-    assertEquals(StringUtf8Coder.of(), source.getDefaultOutputCoder());
-
-    assertEquals(TikaSource.Mode.FILEPATTERN, source.getMode());
-    assertTrue(source.createReader(null) instanceof TikaReader);
-
-    List<? extends TikaSource> sources = source.split(1, null);
-    assertEquals(1, sources.size());
-    TikaSource nextSource = sources.get(0);
-    assertEquals(TikaSource.Mode.SINGLE_FILE, nextSource.getMode());
-    assertEquals(resourcePath, nextSource.getSingleFileMetadata().resourceId().toString());
-  }
-
-  @Test
-  public void testOdtFilesSource() throws Exception {
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-    String resourcePath2 = getClass().getResource("/apache-beam-tika2.odt").getPath();
-    String filePattern = resourcePath.replace("apache-beam-tika1", "*");
-
-    TikaSource source = new TikaSource(TikaIO.read().from(filePattern));
-    assertEquals(StringUtf8Coder.of(), source.getDefaultOutputCoder());
-
-    assertEquals(TikaSource.Mode.FILEPATTERN, source.getMode());
-    assertTrue(source.createReader(null) instanceof TikaSource.FilePatternTikaReader);
-
-    List<? extends TikaSource> sources = source.split(1, null);
-    assertEquals(2, sources.size());
-    TikaSource nextSource = sources.get(0);
-    assertEquals(TikaSource.Mode.SINGLE_FILE, nextSource.getMode());
-    String nextSourceResource = nextSource.getSingleFileMetadata().resourceId().toString();
-    TikaSource nextSource2 = sources.get(1);
-    assertEquals(TikaSource.Mode.SINGLE_FILE, nextSource2.getMode());
-    String nextSourceResource2 = nextSource2.getSingleFileMetadata().resourceId().toString();
-    assertTrue(nextSourceResource.equals(resourcePath) && nextSourceResource2.equals(resourcePath2)
-        || nextSourceResource.equals(resourcePath2) && nextSourceResource2.equals(resourcePath));
-  }
-}


[2/3] beam git commit: Adds ParseResult.failure()

Posted by jk...@apache.org.
Adds ParseResult.failure()

Also groups the tests into fewer methods providing the same coverage.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c76c518b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c76c518b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c76c518b

Branch: refs/heads/master
Commit: c76c518b62eee22334ff77fd15ae4eeae089e828
Parents: ba93dd3
Author: Eugene Kirpichov <ki...@google.com>
Authored: Wed Oct 25 16:57:36 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Oct 26 13:20:39 2017 -0700

----------------------------------------------------------------------
 .../beam/sdk/testing/SuccessOrFailure.java      |  21 +-
 .../beam/sdk/util/SerializableThrowable.java    |  49 +++++
 sdks/java/io/tika/pom.xml                       |   6 +
 .../apache/beam/sdk/io/tika/ParseResult.java    | 104 +++++++---
 .../org/apache/beam/sdk/io/tika/TikaIO.java     | 192 +++++++++--------
 .../beam/sdk/io/tika/ParseResultTest.java       |  54 ++++-
 .../org/apache/beam/sdk/io/tika/TikaIOTest.java | 208 +++++++------------
 .../src/test/resources/apache-beam-tika-pdf.zip | Bin 11685 -> 0 bytes
 .../src/test/resources/apache-beam-tika.pdf     | Bin 12392 -> 0 bytes
 .../src/test/resources/apache-beam-tika1.odt    | Bin 12540 -> 0 bytes
 .../src/test/resources/apache-beam-tika2.odt    | Bin 11412 -> 0 bytes
 .../resources/valid/apache-beam-tika-pdf.zip    | Bin 0 -> 11685 bytes
 .../test/resources/valid/apache-beam-tika.odt   | Bin 0 -> 12540 bytes
 13 files changed, 351 insertions(+), 283 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
index a63bbcc..bac4eb3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/SuccessOrFailure.java
@@ -18,34 +18,17 @@
 package org.apache.beam.sdk.testing;
 
 import com.google.common.base.MoreObjects;
-import java.io.IOException;
-import java.io.ObjectInputStream;
 import java.io.Serializable;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.coders.DefaultCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.util.SerializableThrowable;
 
 /**
  * Output of {@link PAssert}. Passed to a conclude function to act upon.
  */
 @DefaultCoder(SerializableCoder.class)
 public final class SuccessOrFailure implements Serializable {
-  private static final class SerializableThrowable implements Serializable {
-    @Nullable private final Throwable throwable;
-    @Nullable private final StackTraceElement[] stackTrace;
-
-    private SerializableThrowable(@Nullable Throwable t) {
-      this.throwable = t;
-      this.stackTrace = (t == null) ? null : t.getStackTrace();
-    }
-
-    private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
-      is.defaultReadObject();
-      if (throwable != null) {
-        throwable.setStackTrace(stackTrace);
-      }
-    }
-  }
 
   private final boolean isSuccess;
   @Nullable
@@ -68,7 +51,7 @@ public final class SuccessOrFailure implements Serializable {
 
   @Nullable
   public AssertionError assertionError() {
-    return site == null ? null : site.wrap(throwable.throwable);
+    return site == null ? null : site.wrap(throwable.getThrowable());
   }
 
   public static SuccessOrFailure success() {

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
new file mode 100644
index 0000000..4951958
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/SerializableThrowable.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.util;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+
+/**
+ * A wrapper around {@link Throwable} that preserves the stack trace on serialization, unlike
+ * regular {@link Throwable}.
+ */
+public final class SerializableThrowable implements Serializable {
+  @Nullable private final Throwable throwable;
+  @Nullable private final StackTraceElement[] stackTrace;
+
+  public SerializableThrowable(@Nullable Throwable t) {
+    this.throwable = t;
+    this.stackTrace = (t == null) ? null : t.getStackTrace();
+  }
+
+  @Nullable
+  public Throwable getThrowable() {
+    return throwable;
+  }
+
+  private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException {
+    is.defaultReadObject();
+    if (throwable != null) {
+      throwable.setStackTrace(stackTrace);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/pom.xml
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/pom.xml b/sdks/java/io/tika/pom.xml
index d7f7e42..5e48099 100644
--- a/sdks/java/io/tika/pom.xml
+++ b/sdks/java/io/tika/pom.xml
@@ -67,6 +67,12 @@
         </dependency>
 
         <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
           <groupId>org.apache.beam</groupId>
           <artifactId>beam-sdks-java-core</artifactId>
           <classifier>tests</classifier>

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
index 0a77491..f78d603 100644
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/ParseResult.java
@@ -17,61 +17,95 @@
  */
 package org.apache.beam.sdk.io.tika;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Throwables;
 import java.io.Serializable;
 import java.util.Arrays;
-
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.util.SerializableThrowable;
 import org.apache.tika.metadata.Metadata;
 
 /**
- * Tika parse result containing the file location, metadata
- * and content converted to String.
+ * The result of parsing a single file with Tika: contains the file's location, metadata, extracted
+ * text, and optionally an error. If there is an error, the metadata and extracted text may be
+ * partial (i.e. not represent the entire file).
  */
-@SuppressWarnings("serial")
 public class ParseResult implements Serializable {
   private final String fileLocation;
   private final String content;
   private final Metadata metadata;
   private final String[] metadataNames;
+  @Nullable private final SerializableThrowable error;
+
+  public static ParseResult success(String fileLocation, String content, Metadata metadata) {
+    return new ParseResult(fileLocation, content, metadata, null);
+  }
 
-  public ParseResult(String fileLocation, String content) {
-    this(fileLocation, content, new Metadata());
+  public static ParseResult success(String fileLocation, String content) {
+    return new ParseResult(fileLocation, content, new Metadata(), null);
   }
 
-  public ParseResult(String fileLocation, String content, Metadata metadata) {
+  public static ParseResult failure(
+      String fileLocation, String partialContent, Metadata partialMetadata, Throwable error) {
+    return new ParseResult(fileLocation, partialContent, partialMetadata, error);
+  }
+
+  private ParseResult(String fileLocation, String content, Metadata metadata, Throwable error) {
+    checkArgument(fileLocation != null, "fileLocation can not be null");
+    checkArgument(content != null, "content can not be null");
+    checkArgument(metadata != null, "metadata can not be null");
     this.fileLocation = fileLocation;
     this.content = content;
     this.metadata = metadata;
     this.metadataNames = metadata.names();
+    this.error = (error == null) ? null : new SerializableThrowable(error);
+  }
+
+  /** Returns the absolute path to the input file. */
+  public String getFileLocation() {
+    return fileLocation;
+  }
+
+  /** Returns whether this file was parsed successfully. */
+  public boolean isSuccess() {
+    return error == null;
+  }
+
+  /** Returns the parse error, if the file was parsed unsuccessfully. */
+  public Throwable getError() {
+    checkState(error != null, "This is a successful ParseResult");
+    return error.getThrowable();
   }
 
   /**
-   * Gets a file content.
+   * Same as {@link #getError}, but returns the complete stack trace of the error as a {@link
+   * String}.
    */
+  public String getErrorAsString() {
+    return Throwables.getStackTraceAsString(getError());
+  }
+
+  /** Returns the extracted text. May be partial, if this parse result contains a failure. */
   public String getContent() {
     return content;
   }
 
-  /**
-   * Gets a file metadata.
-   */
+  /** Returns the extracted metadata. May be partial, if this parse result contains a failure. */
   public Metadata getMetadata() {
     return metadata;
   }
 
-  /**
-   * Gets a file location.
-   */
-  public String getFileLocation() {
-    return fileLocation;
-  }
-
   @Override
   public int hashCode() {
-    int hashCode = 1;
-    hashCode = 31 * hashCode + fileLocation.hashCode();
-    hashCode = 31 * hashCode + content.hashCode();
-    hashCode = 31 * hashCode + getMetadataHashCode();
-    return hashCode;
+    return Objects.hash(
+        getFileLocation(),
+        getContent(),
+        getMetadataHashCode(),
+        isSuccess() ? "" : Throwables.getStackTraceAsString(getError()));
   }
 
   @Override
@@ -80,14 +114,16 @@ public class ParseResult implements Serializable {
       return false;
     }
 
-    ParseResult pr = (ParseResult) obj;
-    return this.fileLocation.equals(pr.fileLocation)
-      && this.content.equals(pr.content)
-      && this.metadata.equals(pr.metadata);
+    ParseResult other = (ParseResult) obj;
+    return Objects.equals(getFileLocation(), other.getFileLocation())
+        && Objects.equals(getContent(), other.getContent())
+        && Objects.equals(getMetadata(), other.getMetadata())
+        && (isSuccess()
+            ? other.isSuccess()
+            : (!other.isSuccess() && Objects.equals(getErrorAsString(), other.getErrorAsString())));
   }
 
-  //TODO:
-  // Remove this function and use metadata.hashCode() once Apache Tika 1.17 gets released.
+  // TODO: Remove this function and use metadata.hashCode() once Apache Tika 1.17 gets released.
   private int getMetadataHashCode() {
     int hashCode = 0;
     for (String name : metadataNames) {
@@ -95,4 +131,14 @@ public class ParseResult implements Serializable {
     }
     return hashCode;
   }
+
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(this)
+        .add("fileLocation", fileLocation)
+        .add("content", "<" + content.length() + " chars>")
+        .add("metadata", metadata)
+        .add("error", getError() == null ? null : Throwables.getStackTraceAsString(getError()))
+        .toString();
+  }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
index 32353e1..26f116d 100644
--- a/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
+++ b/sdks/java/io/tika/src/main/java/org/apache/beam/sdk/io/tika/TikaIO.java
@@ -17,15 +17,13 @@
  */
 package org.apache.beam.sdk.io.tika;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.auto.value.AutoValue;
-
 import java.io.InputStream;
 import java.nio.channels.Channels;
-
 import javax.annotation.Nullable;
-
 import org.apache.beam.sdk.annotations.Experimental;
 import org.apache.beam.sdk.io.Compression;
 import org.apache.beam.sdk.io.FileIO;
@@ -49,68 +47,60 @@ import org.apache.tika.parser.Parser;
 import org.apache.tika.sax.ToTextContentHandler;
 import org.xml.sax.ContentHandler;
 
-
-
-
 /**
- * A collection of {@link PTransform} transforms for parsing arbitrary files using Apache Tika.
- * Files in many well known text, binary or scientific formats can be processed.
+ * Transforms for parsing arbitrary files using <a href="https://tika.apache.org/">Apache Tika</a>.
+ *
+ * <p>Tika is able to extract text and metadata from files in many well known text, binary and
+ * scientific formats.
  *
- * <p>{@link TikaIO.Parse} and {@link TikaIO.ParseAll} parse the files and return
- * a {@link PCollection} containing one {@link ParseResult} per each file.
+ * <p>The entry points are {@link #parse} and {@link #parseFiles}. They parse a set of files and
+ * return a {@link PCollection} containing one {@link ParseResult} per each file. {@link #parse}
+ * implements the common case of parsing all files matching a single filepattern, while {@link
+ * #parseFiles} should be used for all use cases requiring more control, in combination with {@link
+ * FileIO#match} and {@link FileIO#readMatches} (see their respective documentation).
  *
- * <p>Combine {@link TikaIO.ParseAll} with {@link FileIO.Match}
- * and {@link FileIO.ReadMatches} to match, read and parse the files.
+ * <p>{@link #parse} does not automatically uncompress compressed files: they are passed to Tika
+ * as-is.
  *
- * <p>Example:
+ * <p>It's possible that some files will partially or completely fail to parse. In that case, the
+ * respective {@link ParseResult} will be marked unsuccessful (see {@link ParseResult#isSuccess})
+ * and will contain the error, available via {@link ParseResult#getError}.
+ *
+ * <p>Example: using {@link #parse} to parse all PDF files in a directory on GCS.
  *
  * <pre>{@code
  * Pipeline p = ...;
  *
- * // A simple parse of a local PDF file (only runs locally):
  * PCollection<ParseResult> results =
- *   p.apply(FileIO.match().filepattern("/local/path/to/file.pdf"))
- *    .apply(FileIO.readMatches())
- *    .apply(TikaIO.parseFiles());
+ *   p.apply(TikaIO.parse().filepattern("gs://my-bucket/files/*.pdf"));
  * }</pre>
  *
- * <p>Use {@link TikaIO.Parse} to match, read and parse the files in simple cases.
- *
- * <p>Example:
+ * <p>Example: using {@link #parseFiles} in combination with {@link FileIO} to continuously parse
+ * new PDF files arriving into the directory.
  *
  * <pre>{@code
  * Pipeline p = ...;
  *
- * // A simple parse of a local PDF file (only runs locally):
  * PCollection<ParseResult> results =
- *   p.apply(TikaIO.parseAll().filepattern("/local/path/to/file.pdf"));
+ *   p.apply(FileIO.match().filepattern("gs://my-bucket/files/*.pdf")
+ *       .continuously(...))
+ *    .apply(FileIO.readMatches())
+ *    .apply(TikaIO.parseFiles());
  * }</pre>
- *
- * <b>Warning:</b> the API of this IO is likely to change in the next release.
  */
 @Experimental(Experimental.Kind.SOURCE_SINK)
 public class TikaIO {
-
-  /**
-   * A {@link PTransform} that matches and parses the files
-   * and returns a bounded {@link PCollection} of {@link ParseResult}.
-   */
+  /** Parses files matching a given filepattern. */
   public static Parse parse() {
-    return new AutoValue_TikaIO_Parse.Builder()
-        .build();
+    return new AutoValue_TikaIO_Parse.Builder().build();
   }
 
-  /**
-   * A {@link PTransform} that accepts a {@link PCollection} of {@link ReadableFile}
-   * and returns a {@link PCollection} of {@link ParseResult}.
-   */
-  public static ParseAll parseAll() {
-    return new AutoValue_TikaIO_ParseAll.Builder()
-        .build();
+  /** Parses files in a {@link PCollection} of {@link ReadableFile}. */
+  public static ParseFiles parseFiles() {
+    return new AutoValue_TikaIO_ParseFiles.Builder().build();
   }
 
   /** Implementation of {@link #parse}. */
-  @SuppressWarnings("serial")
   @AutoValue
   public abstract static class Parse extends PTransform<PBegin, PCollection<ParseResult>> {
     @Nullable
@@ -139,8 +129,7 @@ public class TikaIO {
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
 
-      builder
-        .addIfNotNull(
+      builder.addIfNotNull(
           DisplayData.item("filePattern", getFilepattern()).withLabel("File Pattern"));
     }
 
@@ -149,65 +138,68 @@ public class TikaIO {
       return input
           .apply(FileIO.match().filepattern(getFilepattern()))
           .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
-          .apply(parseAll());
+          .apply(parseFiles());
     }
   }
 
-  /** Implementation of {@link #parseAll}. */
-  @SuppressWarnings("serial")
+  /** Implementation of {@link #parseFiles}. */
   @AutoValue
-  public abstract static class ParseAll extends
-    PTransform<PCollection<ReadableFile>, PCollection<ParseResult>> {
+  public abstract static class ParseFiles
+      extends PTransform<PCollection<ReadableFile>, PCollection<ParseResult>> {
+
+    @Nullable
+    abstract ValueProvider<String> getTikaConfigPath();
 
-    @Nullable abstract ValueProvider<String> getTikaConfigPath();
-    @Nullable abstract Metadata getInputMetadata();
+    @Nullable
+    abstract String getContentTypeHint();
+
+    @Nullable
+    abstract Metadata getInputMetadata();
 
     abstract Builder toBuilder();
 
     @AutoValue.Builder
     abstract static class Builder {
       abstract Builder setTikaConfigPath(ValueProvider<String> tikaConfigPath);
+
+      abstract Builder setContentTypeHint(String contentTypeHint);
+
       abstract Builder setInputMetadata(Metadata metadata);
 
-      abstract ParseAll build();
+      abstract ParseFiles build();
     }
 
     /**
-     * Returns a new transform which will use the custom TikaConfig.
+     * Uses the given <a
+     * href="https://tika.apache.org/1.16/configuring.html#Using_a_Tika_Configuration_XML_file">Tika
+     * Configuration XML file</a>.
      */
-    public ParseAll withTikaConfigPath(String tikaConfigPath) {
-      checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
+    public ParseFiles withTikaConfigPath(String tikaConfigPath) {
+      checkArgument(tikaConfigPath != null, "tikaConfigPath can not be null.");
       return withTikaConfigPath(StaticValueProvider.of(tikaConfigPath));
     }
 
-    /** Same as {@code with(tikaConfigPath)}, but accepting a {@link ValueProvider}. */
-    public ParseAll withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
-      checkNotNull(tikaConfigPath, "TikaConfigPath cannot be empty.");
-      return toBuilder()
-          .setTikaConfigPath(tikaConfigPath)
-          .build();
+    /** Like {@code with(tikaConfigPath)}. */
+    public ParseFiles withTikaConfigPath(ValueProvider<String> tikaConfigPath) {
+      checkArgument(tikaConfigPath != null, "tikaConfigPath can not be null.");
+      return toBuilder().setTikaConfigPath(tikaConfigPath).build();
     }
 
     /**
-     * Returns a new transform which will use the provided content type hint
-     * to make the file parser detection more efficient.
+     * Sets a content type hint to make the file parser detection more efficient. Overrides the
+     * content type hint in {@link #withInputMetadata}, if any.
      */
-    public ParseAll withContentTypeHint(String contentType) {
-      checkNotNull(contentType, "ContentType cannot be empty.");
-      Metadata metadata = new Metadata();
-      metadata.add(Metadata.CONTENT_TYPE, contentType);
-      return withInputMetadata(metadata);
+    public ParseFiles withContentTypeHint(String contentTypeHint) {
+      checkNotNull(contentTypeHint, "contentTypeHint can not be null.");
+      return toBuilder().setContentTypeHint(contentTypeHint).build();
     }
 
-    /**
-     * Returns a new transform which will use the provided input metadata
-     * for parsing the files.
-     */
-    public ParseAll withInputMetadata(Metadata metadata) {
+    /** Sets the input metadata for {@link Parser#parse}. */
+    public ParseFiles withInputMetadata(Metadata metadata) {
       Metadata inputMetadata = this.getInputMetadata();
       if (inputMetadata != null) {
         for (String name : metadata.names()) {
-            inputMetadata.set(name, metadata.get(name));
+          inputMetadata.set(name, metadata.get(name));
         }
       } else {
         inputMetadata = metadata;
@@ -225,27 +217,25 @@ public class TikaIO {
       super.populateDisplayData(builder);
 
       if (getTikaConfigPath() != null) {
-        String tikaConfigPathDisplay = getTikaConfigPath().isAccessible()
-          ? getTikaConfigPath().get() : getTikaConfigPath().toString();
-        builder.add(DisplayData.item("tikaConfigPath", tikaConfigPathDisplay)
-            .withLabel("TikaConfig Path"));
+        builder.add(
+            DisplayData.item("tikaConfigPath", getTikaConfigPath()).withLabel("TikaConfig Path"));
       }
       Metadata metadata = getInputMetadata();
       if (metadata != null) {
         //TODO: use metadata.toString() only without a trim() once Apache Tika 1.17 gets released
-        builder
-            .add(DisplayData.item("inputMetadata", metadata.toString().trim())
-            .withLabel("Input Metadata"));
+        builder.add(
+            DisplayData.item("inputMetadata", metadata.toString().trim())
+                .withLabel("Input Metadata"));
       }
+      builder.addIfNotNull(
+          DisplayData.item("contentTypeHint", getContentTypeHint()).withLabel("Content type hint"));
     }
 
     private static class ParseToStringFn extends DoFn<ReadableFile, ParseResult> {
+      private final ParseFiles spec;
+      private transient TikaConfig tikaConfig;
 
-      private static final long serialVersionUID = 6837207505313720989L;
-      private final TikaIO.ParseAll spec;
-      private TikaConfig tikaConfig;
-
-      ParseToStringFn(TikaIO.ParseAll spec) {
+      ParseToStringFn(ParseFiles spec) {
         this.spec = spec;
       }
 
@@ -254,8 +244,7 @@ public class TikaIO {
         if (spec.getTikaConfigPath() != null) {
           ResourceId configResource =
               FileSystems.matchSingleFileSpec(spec.getTikaConfigPath().get()).resourceId();
-          tikaConfig = new TikaConfig(
-                           Channels.newInputStream(FileSystems.open(configResource)));
+          tikaConfig = new TikaConfig(Channels.newInputStream(FileSystems.open(configResource)));
         }
       }
 
@@ -264,21 +253,30 @@ public class TikaIO {
         ReadableFile file = c.element();
         InputStream stream = Channels.newInputStream(file.open());
         try (InputStream tikaStream = TikaInputStream.get(stream)) {
+          Parser parser =
+              tikaConfig == null ? new AutoDetectParser() : new AutoDetectParser(tikaConfig);
 
-          final Parser parser = tikaConfig == null
-              ? new AutoDetectParser() : new AutoDetectParser(tikaConfig);
-
-          final ParseContext context = new ParseContext();
+          ParseContext context = new ParseContext();
           context.set(Parser.class, parser);
-          Metadata tikaMetadata = spec.getInputMetadata() != null
-            ? spec.getInputMetadata() : new org.apache.tika.metadata.Metadata();
-
+          Metadata tikaMetadata =
+              spec.getInputMetadata() != null
+                  ? spec.getInputMetadata()
+                  : new org.apache.tika.metadata.Metadata();
+          if (spec.getContentTypeHint() != null) {
+            tikaMetadata.set(Metadata.CONTENT_TYPE, spec.getContentTypeHint());
+          }
+
+          String location = file.getMetadata().resourceId().toString();
+          ParseResult res;
           ContentHandler tikaHandler = new ToTextContentHandler();
-          parser.parse(tikaStream, tikaHandler, tikaMetadata, context);
-
-          c.output(new ParseResult(file.getMetadata().resourceId().toString(),
-              tikaHandler.toString(),
-              tikaMetadata));
+          try {
+            parser.parse(tikaStream, tikaHandler, tikaMetadata, context);
+            res = ParseResult.success(location, tikaHandler.toString(), tikaMetadata);
+          } catch (Exception e) {
+            res = ParseResult.failure(location, tikaHandler.toString(), tikaMetadata, e);
+          }
+
+          c.output(res);
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
index fd86152..95bcee0 100644
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/ParseResultTest.java
@@ -17,20 +17,60 @@
  */
 package org.apache.beam.sdk.io.tika;
 
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.not;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
 import org.apache.tika.metadata.Metadata;
 import org.junit.Test;
 
-/**
- * Tests ParseResult.
- */
+/** Tests {@link ParseResult}. */
 public class ParseResultTest {
   @Test
   public void testEqualsAndHashCode() {
-    ParseResult p1 = new ParseResult("a.txt", "hello", getMetadata());
-    ParseResult p2 = new ParseResult("a.txt", "hello", getMetadata());
-    assertEquals(p1, p2);
-    assertEquals(p1.hashCode(), p2.hashCode());
+    ParseResult successBase = ParseResult.success("a.txt", "hello", getMetadata());
+    ParseResult successSame = ParseResult.success("a.txt", "hello", getMetadata());
+    ParseResult successDifferentName = ParseResult.success("b.txt", "hello", getMetadata());
+    ParseResult successDifferentContent = ParseResult.success("a.txt", "goodbye", getMetadata());
+    ParseResult successDifferentMetadata = ParseResult.success("a.txt", "hello", new Metadata());
+
+    RuntimeException oops = new RuntimeException("oops");
+    ParseResult failureBase = ParseResult.failure("a.txt", "", new Metadata(), oops);
+    ParseResult failureSame = ParseResult.failure("a.txt", "", new Metadata(), oops);
+    ParseResult failureDifferentName = ParseResult.failure("b.txt", "", new Metadata(), oops);
+    ParseResult failureDifferentContent =
+        ParseResult.failure("b.txt", "partial", new Metadata(), oops);
+    ParseResult failureDifferentMetadata = ParseResult.failure("b.txt", "", getMetadata(), oops);
+    ParseResult failureDifferentError =
+        ParseResult.failure("a.txt", "", new Metadata(), new RuntimeException("eek"));
+
+    assertEquals(successBase, successSame);
+    assertEquals(successBase.hashCode(), successSame.hashCode());
+
+    assertThat(successDifferentName, not(equalTo(successBase)));
+    assertThat(successDifferentContent, not(equalTo(successBase)));
+    assertThat(successDifferentMetadata, not(equalTo(successBase)));
+
+    assertThat(successDifferentName.hashCode(), not(equalTo(successBase.hashCode())));
+    assertThat(successDifferentContent.hashCode(), not(equalTo(successBase.hashCode())));
+    assertThat(successDifferentMetadata.hashCode(), not(equalTo(successBase.hashCode())));
+
+    assertThat(failureBase, not(equalTo(successBase)));
+    assertThat(successBase, not(equalTo(failureBase)));
+
+    assertEquals(failureBase, failureSame);
+    assertEquals(failureBase.hashCode(), failureSame.hashCode());
+
+    assertThat(failureDifferentName, not(equalTo(failureBase)));
+    assertThat(failureDifferentError, not(equalTo(failureBase)));
+    assertThat(failureDifferentContent, not(equalTo(failureBase)));
+    assertThat(failureDifferentMetadata, not(equalTo(failureBase)));
+
+    assertThat(failureDifferentName.hashCode(), not(equalTo(failureBase.hashCode())));
+    assertThat(failureDifferentError.hashCode(), not(equalTo(failureBase.hashCode())));
+    assertThat(failureDifferentContent.hashCode(), not(equalTo(failureBase.hashCode())));
+    assertThat(failureDifferentMetadata.hashCode(), not(equalTo(failureBase.hashCode())));
   }
 
   static Metadata getMetadata() {

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
index a985b0a..1c95e9f 100644
--- a/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
+++ b/sdks/java/io/tika/src/test/java/org/apache/beam/sdk/io/tika/TikaIOTest.java
@@ -18,124 +18,109 @@
 package org.apache.beam.sdk.io.tika;
 
 import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
-import static org.hamcrest.Matchers.isA;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
-
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
 import org.apache.beam.sdk.io.Compression;
 import org.apache.beam.sdk.io.FileIO;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.metadata.Metadata;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-/**
- * Tests TikaInput.
- */
-public class TikaIOTest {
-  private static final String PDF_FILE =
-      "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
-      + "Combining\n\nApache Beam\n\nand\n\nApache Tika\n\ncan help to ingest\n\n"
-      + "the content from the files\n\nin most known formats.\n\n\n";
 
+/** Tests for {@link TikaIO}. */
+public class TikaIOTest implements Serializable {
   private static final String PDF_ZIP_FILE =
       "\n\n\n\n\n\n\n\napache-beam-tika.pdf\n\n\nCombining\n\n\nApache Beam\n\n\n"
-      + "and\n\n\nApache Tika\n\n\ncan help to ingest\n\n\nthe content from the files\n\n\n"
-      + "in most known formats.\n\n\n\n\n\n\n";
+          + "and\n\n\nApache Tika\n\n\ncan help to ingest\n\n\nthe content from the files\n\n\n"
+          + "in most known formats.\n\n\n\n\n\n\n";
 
   private static final String ODT_FILE =
       "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
-      + "Combining\nApache Beam\nand\nApache Tika\ncan help to ingest\nthe content from the"
-      + " files\nin most known formats.\n";
-
-  private static final String ODT_FILE2 =
-      "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
-      + "Open Office\nPDF\nExcel\nText\nScientific\nand other formats\nare supported.\n";
+          + "Combining\nApache Beam\nand\nApache Tika\ncan help to ingest\nthe content from the"
+          + " files\nin most known formats.\n";
 
   @Rule
-  public TestPipeline p = TestPipeline.create();
-  @Rule
-  public ExpectedException thrown = ExpectedException.none();
-
-  @Test
-  public void testParsePdfFile() throws IOException {
-
-    String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
-
-    doTestParse(resourcePath, new ParseResult(resourcePath, PDF_FILE));
-  }
-
-  private void doTestParse(String resourcePath, ParseResult... expectedResults)
-      throws IOException {
-     PCollection<ParseResult> output =
-         p.apply("ParseAll", TikaIO.parse().filepattern(resourcePath))
-         .apply(ParDo.of(new FilterMetadataFn()));
-     PAssert.that(output).containsInAnyOrder(expectedResults);
-     p.run();
-  }
-
-  @Test
-  public void testParseAllPdfFile() throws IOException {
-
-    String resourcePath = getClass().getResource("/apache-beam-tika.pdf").getPath();
-
-    doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_FILE));
-  }
-
-  @Test
-  public void testParseAllZipPdfFile() throws IOException {
-
-    String resourcePath = getClass().getResource("/apache-beam-tika-pdf.zip").getPath();
-
-    doTestParseAll(resourcePath, new ParseResult(resourcePath, PDF_ZIP_FILE));
-  }
+  public transient TestPipeline p = TestPipeline.create();
 
   @Test
-  public void testParseAllOdtFile() throws IOException {
-
-    String resourcePath = getClass().getResource("/apache-beam-tika1.odt").getPath();
-
-    doTestParseAll(resourcePath, new ParseResult(resourcePath, ODT_FILE, getOdtMetadata()));
+  public void testParseAndParseFiles() throws IOException {
+    Path root =
+        Paths.get(getClass().getResource("/valid/apache-beam-tika.odt").getPath()).getParent();
+
+    List<ParseResult> expected =
+        Arrays.asList(
+            ParseResult.success(
+                root.resolve("apache-beam-tika.odt").toString(), ODT_FILE, getOdtMetadata()),
+            ParseResult.success(root.resolve("apache-beam-tika-pdf.zip").toString(), PDF_ZIP_FILE));
+
+    PCollection<ParseResult> parse =
+        p.apply("Parse", TikaIO.parse().filepattern(root.resolve("*").toString()))
+            .apply("FilterParse", ParDo.of(new FilterMetadataFn()));
+    PAssert.that(parse).containsInAnyOrder(expected);
+
+    PCollection<ParseResult> parseFiles =
+        p.apply("ParseFiles", FileIO.match().filepattern(root.resolve("*").toString()))
+            .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
+            .apply(TikaIO.parseFiles())
+            .apply("FilterParseFiles", ParDo.of(new FilterMetadataFn()));
+    PAssert.that(parseFiles).containsInAnyOrder(expected);
+    p.run();
   }
 
-  @Test
-  public void testParseAllOdtFiles() throws IOException {
-    String resourcePath1 = getClass().getResource("/apache-beam-tika1.odt").getPath();
-    String resourcePath2 = getClass().getResource("/apache-beam-tika2.odt").getPath();
-    String resourcePath = resourcePath1.replace("apache-beam-tika1", "*");
-
-    doTestParseAll(resourcePath, new ParseResult(resourcePath1, ODT_FILE, getOdtMetadata()),
-        new ParseResult(resourcePath2, ODT_FILE2));
+  private static Metadata getOdtMetadata() {
+    Metadata m = new Metadata();
+    m.set("Author", "BeamTikaUser");
+    return m;
   }
 
-  private void doTestParseAll(String resourcePath, ParseResult... expectedResults)
-     throws IOException {
-    PCollection<ParseResult> output =
-        p.apply("ParseFiles", FileIO.match().filepattern(resourcePath))
-        .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
-        .apply(TikaIO.parseAll())
-        .apply(ParDo.of(new FilterMetadataFn()));
-    PAssert.that(output).containsInAnyOrder(expectedResults);
-    p.run();
+  private static class FilterMetadataFn extends DoFn<ParseResult, ParseResult> {
+    @ProcessElement
+    public void processElement(ProcessContext c) {
+      ParseResult result = c.element();
+      Metadata m = new Metadata();
+      // Files contain many metadata properties. This function drops all but the "Author"
+      // property manually added to "apache-beam-tika.odt" resource only to make
+      // the tests simpler
+      if (result.getFileLocation().endsWith("valid/apache-beam-tika.odt")) {
+        m.set("Author", result.getMetadata().get("Author"));
+      }
+      ParseResult newResult = ParseResult.success(result.getFileLocation(), result.getContent(), m);
+      c.output(newResult);
+    }
   }
 
   @Test
-  public void testParseAllDamagedPdfFile() throws IOException {
-    thrown.expectCause(isA(TikaException.class));
-    String resourcePath = getClass().getResource("/damaged.pdf").getPath();
-
-    p.apply("ParseInvalidPdfFile", FileIO.match().filepattern(resourcePath))
-      .apply(FileIO.readMatches())
-      .apply(TikaIO.parseAll());
+  public void testParseDamagedPdfFile() throws IOException {
+    String path = getClass().getResource("/damaged.pdf").getPath();
+    PCollection<ParseResult> res = p.apply("ParseInvalidPdfFile", TikaIO.parse().filepattern(path));
+
+    PAssert.thatSingleton(res)
+        .satisfies(
+            new SerializableFunction<ParseResult, Void>() {
+              @Override
+              public Void apply(ParseResult input) {
+                assertEquals(path, input.getFileLocation());
+                assertFalse(input.isSuccess());
+                assertTrue(input.getError() instanceof TikaException);
+                return null;
+              }
+            });
     p.run();
   }
 
@@ -150,54 +135,15 @@ public class TikaIOTest {
   }
 
   @Test
-  public void testParseAllDisplayData() {
-    TikaIO.ParseAll parseAll = TikaIO.parseAll()
-        .withTikaConfigPath("tikaconfigpath")
-        .withContentTypeHint("application/pdf");
-
-    DisplayData displayData = DisplayData.from(parseAll);
-
-    assertThat(displayData, hasDisplayItem("tikaConfigPath", "tikaconfigpath"));
-    assertThat(displayData, hasDisplayItem("inputMetadata",
-        "Content-Type=application/pdf"));
-    assertEquals(2, displayData.items().size());
-  }
-
-  @Test
-  public void testParseAllDisplayDataWithCustomOptions() {
-    TikaIO.ParseAll parseAll = TikaIO.parseAll()
-        .withTikaConfigPath("/tikaConfigPath")
-        .withContentTypeHint("application/pdf");
+  public void testParseFilesDisplayData() {
+    TikaIO.ParseFiles parseFiles =
+        TikaIO.parseFiles()
+            .withTikaConfigPath("/tikaConfigPath")
+            .withContentTypeHint("application/pdf");
 
-    DisplayData displayData = DisplayData.from(parseAll);
+    DisplayData displayData = DisplayData.from(parseFiles);
 
     assertThat(displayData, hasDisplayItem("tikaConfigPath", "/tikaConfigPath"));
-    assertThat(displayData, hasDisplayItem("inputMetadata",
-        "Content-Type=application/pdf"));
-    assertEquals(2, displayData.items().size());
-  }
-
-  static class FilterMetadataFn extends DoFn<ParseResult, ParseResult> {
-    private static final long serialVersionUID = 6338014219600516621L;
-
-    @ProcessElement
-    public void processElement(ProcessContext c) {
-      ParseResult result = c.element();
-      Metadata m = new Metadata();
-      // Files contain many metadata properties. This function drops all but the "Author"
-      // property manually added to "apache-beam-tika1.odt" resource only to make
-      // the tests simpler
-      if (result.getFileLocation().endsWith("apache-beam-tika1.odt")) {
-          m.set("Author", result.getMetadata().get("Author"));
-      }
-      ParseResult newResult = new ParseResult(result.getFileLocation(), result.getContent(), m);
-      c.output(newResult);
-    }
-  }
-
-  static Metadata getOdtMetadata() {
-    Metadata m = new Metadata();
-    m.set("Author", "BeamTikaUser");
-    return m;
+    assertThat(displayData, hasDisplayItem("contentTypeHint", "application/pdf"));
   }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip b/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip
deleted file mode 100644
index 4c0e0ef..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika-pdf.zip and /dev/null differ

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf b/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf
deleted file mode 100644
index d3c7f14..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika.pdf and /dev/null differ

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt b/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt
deleted file mode 100644
index 87c5577..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika1.odt and /dev/null differ

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt b/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt
deleted file mode 100644
index a0ff320..0000000
Binary files a/sdks/java/io/tika/src/test/resources/apache-beam-tika2.odt and /dev/null differ

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip
new file mode 100644
index 0000000..4c0e0ef
Binary files /dev/null and b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika-pdf.zip differ

http://git-wip-us.apache.org/repos/asf/beam/blob/c76c518b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt
----------------------------------------------------------------------
diff --git a/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt
new file mode 100644
index 0000000..87c5577
Binary files /dev/null and b/sdks/java/io/tika/src/test/resources/valid/apache-beam-tika.odt differ


[3/3] beam git commit: This closes #3835: [BEAM-2994] Many improvements to TikaIO

Posted by jk...@apache.org.
This closes #3835: [BEAM-2994] Many improvements to TikaIO


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/0c221137
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/0c221137
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/0c221137

Branch: refs/heads/master
Commit: 0c2211375d1fc4abebda7aa7cf7960e7b09fdc1b
Parents: e01c78d c76c518
Author: Eugene Kirpichov <ki...@google.com>
Authored: Thu Oct 26 13:21:07 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Thu Oct 26 13:21:07 2017 -0700

----------------------------------------------------------------------
 .../beam/sdk/testing/SuccessOrFailure.java      |  21 +-
 .../beam/sdk/util/SerializableThrowable.java    |  49 ++
 sdks/java/io/tika/pom.xml                       |  16 +-
 .../apache/beam/sdk/io/tika/ParseResult.java    | 144 ++++++
 .../org/apache/beam/sdk/io/tika/TikaIO.java     | 394 ++++++++--------
 .../apache/beam/sdk/io/tika/TikaOptions.java    |  78 ----
 .../org/apache/beam/sdk/io/tika/TikaSource.java | 466 -------------------
 .../beam/sdk/io/tika/ParseResultTest.java       |  83 ++++
 .../org/apache/beam/sdk/io/tika/TikaIOTest.java | 290 ++++--------
 .../apache/beam/sdk/io/tika/TikaReaderTest.java |  82 ----
 .../apache/beam/sdk/io/tika/TikaSourceTest.java |  73 ---
 .../src/test/resources/apache-beam-tika-pdf.zip | Bin 11685 -> 0 bytes
 .../src/test/resources/apache-beam-tika.pdf     | Bin 12392 -> 0 bytes
 .../src/test/resources/apache-beam-tika1.odt    | Bin 12540 -> 0 bytes
 .../src/test/resources/apache-beam-tika2.odt    | Bin 11412 -> 0 bytes
 .../resources/valid/apache-beam-tika-pdf.zip    | Bin 0 -> 11685 bytes
 .../test/resources/valid/apache-beam-tika.odt   | Bin 0 -> 12540 bytes
 17 files changed, 556 insertions(+), 1140 deletions(-)
----------------------------------------------------------------------