You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jk...@apache.org on 2017/03/28 15:51:43 UTC

[1/6] beam git commit: Move a few more functions into BigQueryHelper

Repository: beam
Updated Branches:
  refs/heads/master 434eadb53 -> 1339dd706


Move a few more functions into BigQueryHelper


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2381dee4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2381dee4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2381dee4

Branch: refs/heads/master
Commit: 2381dee48faf52882b873597d49b9490967f6adb
Parents: d6ef010
Author: Reuven Lax <re...@google.com>
Authored: Sat Mar 18 13:03:25 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:15 2017 -0700

----------------------------------------------------------------------
 .../sdk/io/gcp/bigquery/BigQueryHelpers.java    | 40 +++++++++++-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    | 67 ++------------------
 2 files changed, 44 insertions(+), 63 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/2381dee4/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 37ff124..c5156e9 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -24,6 +24,7 @@ import com.google.api.services.bigquery.model.Job;
 import com.google.api.services.bigquery.model.JobStatus;
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableSchema;
+import com.google.cloud.hadoop.util.ApiErrorExtractor;
 import com.google.common.annotations.VisibleForTesting;
 
 import java.io.IOException;
@@ -196,6 +197,43 @@ class BigQueryHelpers {
     }
   }
 
+  static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
+    try {
+      datasetService.getDataset(table.getProjectId(), table.getDatasetId());
+    } catch (Exception e) {
+      ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
+      if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
+        throw new IllegalArgumentException(
+            String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", toTableSpec(table)), e);
+      } else if (e instanceof  RuntimeException) {
+        throw (RuntimeException) e;
+      } else {
+        throw new RuntimeException(
+            String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "dataset",
+                toTableSpec(table)),
+            e);
+      }
+    }
+  }
+
+  static void verifyTablePresence(DatasetService datasetService, TableReference table) {
+    try {
+      datasetService.getTable(table);
+    } catch (Exception e) {
+      ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
+      if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
+        throw new IllegalArgumentException(
+            String.format(RESOURCE_NOT_FOUND_ERROR, "table", toTableSpec(table)), e);
+      } else if (e instanceof  RuntimeException) {
+        throw (RuntimeException) e;
+      } else {
+        throw new RuntimeException(
+            String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "table",
+                toTableSpec(table)), e);
+      }
+    }
+  }
+
   @VisibleForTesting
   static class JsonSchemaToTableSchema
       implements SerializableFunction<String, TableSchema> {
@@ -283,7 +321,7 @@ class BigQueryHelpers {
       implements SerializableFunction<String, TableReference> {
     private final String executingProject;
 
-    public CreateJsonTableRefFromUuid(String executingProject) {
+    CreateJsonTableRefFromUuid(String executingProject) {
       this.executingProject = executingProject;
     }
 

http://git-wip-us.apache.org/repos/asf/beam/blob/2381dee4/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 4917083..cc6ec09 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -29,22 +29,17 @@ import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.api.services.bigquery.model.TableSchema;
 import com.google.auto.value.AutoValue;
-import com.google.cloud.hadoop.util.ApiErrorExtractor;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
-import com.google.common.io.CountingOutputStream;
+
 import java.io.IOException;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
-import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.List;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 
 import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
 import org.apache.beam.sdk.coders.TableRowJsonCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.BoundedSource;
@@ -70,7 +65,6 @@ import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.util.GcsUtil.GcsUtilFactory;
 import org.apache.beam.sdk.util.IOChannelFactory;
 import org.apache.beam.sdk.util.IOChannelUtils;
-import org.apache.beam.sdk.util.MimeTypes;
 import org.apache.beam.sdk.util.Transport;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
 import org.apache.beam.sdk.values.PBegin;
@@ -229,17 +223,6 @@ public class BigQueryIO {
 
   static final Pattern TABLE_SPEC = Pattern.compile(DATASET_TABLE_REGEXP);
 
-  private static final String RESOURCE_NOT_FOUND_ERROR =
-      "BigQuery %1$s not found for table \"%2$s\" . Please create the %1$s before pipeline"
-          + " execution. If the %1$s is created by an earlier stage of the pipeline, this"
-          + " validation can be disabled using #withoutValidation.";
-
-  private static final String UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR =
-      "Unable to confirm BigQuery %1$s presence for table \"%2$s\". If the %1$s is created by"
-          + " an earlier stage of the pipeline, this validation can be disabled using"
-          + " #withoutValidation.";
-
-
   /**
    * A formatting function that maps a TableRow to itself. This allows sending a
    * {@code PCollection<TableRow>} directly to BigQueryIO.Write.
@@ -451,8 +434,8 @@ public class BigQueryIO {
         checkState(table.isAccessible(), "Cannot call validate if table is dynamically set.");
         // Check for source table presence for early failure notification.
         DatasetService datasetService = getBigQueryServices().getDatasetService(bqOptions);
-        verifyDatasetPresence(datasetService, table.get());
-        verifyTablePresence(datasetService, table.get());
+        BigQueryHelpers.verifyDatasetPresence(datasetService, table.get());
+        BigQueryHelpers.verifyTablePresence(datasetService, table.get());
       } else if (getValidate() && getQuery() != null) {
         checkState(getQuery().isAccessible(), "Cannot call validate if query is dynamically set.");
         JobService jobService = getBigQueryServices().getJobService(bqOptions);
@@ -953,9 +936,9 @@ public class BigQueryIO {
         // Note that a presence check can fail when the table or dataset is created by an earlier
         // stage of the pipeline. For these cases the #withoutValidation method can be used to
         // disable the check.
-        verifyDatasetPresence(datasetService, table);
+        BigQueryHelpers.verifyDatasetPresence(datasetService, table);
         if (getCreateDisposition() == BigQueryIO.Write.CreateDisposition.CREATE_NEVER) {
-          verifyTablePresence(datasetService, table);
+          BigQueryHelpers.verifyTablePresence(datasetService, table);
         }
         if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
           BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table);
@@ -1085,46 +1068,6 @@ public class BigQueryIO {
 
   }
 
-  private static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
-    try {
-      datasetService.getDataset(table.getProjectId(), table.getDatasetId());
-    } catch (Exception e) {
-      ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
-      if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
-        throw new IllegalArgumentException(
-            String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", BigQueryHelpers.toTableSpec(table)),
-            e);
-      } else if (e instanceof  RuntimeException) {
-        throw (RuntimeException) e;
-      } else {
-        throw new RuntimeException(
-            String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "dataset",
-                BigQueryHelpers.toTableSpec(table)),
-            e);
-      }
-    }
-  }
-
-  private static void verifyTablePresence(DatasetService datasetService, TableReference table) {
-    try {
-      datasetService.getTable(table);
-    } catch (Exception e) {
-      ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
-      if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
-        throw new IllegalArgumentException(
-            String.format(RESOURCE_NOT_FOUND_ERROR, "table", BigQueryHelpers.toTableSpec(table)),
-            e);
-      } else if (e instanceof  RuntimeException) {
-        throw (RuntimeException) e;
-      } else {
-        throw new RuntimeException(
-            String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "table",
-                BigQueryHelpers.toTableSpec(table)),
-            e);
-      }
-    }
-  }
-
   /**
    * Clear the cached map of created tables. Used for testing.
    */


[5/6] beam git commit: Refactor BigQueryIO helper methods into a BigQueryHelpers class. Move helper transforms for BigQueryIO.Read into individual files. Change private to package in BigQueryHelpers.

Posted by jk...@apache.org.
Refactor BigQueryIO helper methods into a BigQueryHelpers class.
Move helper transforms for BigQueryIO.Read into individual files.
Change private to package in BigQueryHelpers.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2cc2e81f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2cc2e81f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2cc2e81f

Branch: refs/heads/master
Commit: 2cc2e81f241e1d5d590f12d4a8491273908e2cce
Parents: 434eadb
Author: Reuven Lax <re...@google.com>
Authored: Fri Mar 17 15:15:16 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:15 2017 -0700

----------------------------------------------------------------------
 .../sdk/io/gcp/bigquery/BigQueryHelpers.java    | 243 ++++++
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    | 853 ++-----------------
 .../io/gcp/bigquery/BigQueryQuerySource.java    | 186 ++++
 .../io/gcp/bigquery/BigQueryServicesImpl.java   |   2 +-
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 178 ++++
 .../io/gcp/bigquery/BigQueryTableSource.java    |  86 ++
 .../io/gcp/bigquery/PassThroughThenCleanup.java |  66 ++
 .../sdk/io/gcp/bigquery/TransformingSource.java | 118 +++
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |  54 +-
 .../sdk/io/gcp/bigquery/BigQueryUtilTest.java   |  12 +-
 10 files changed, 970 insertions(+), 828 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
new file mode 100644
index 0000000..9fba938
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -0,0 +1,243 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobStatus;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+
+
+/**
+ * A set of helper functions and classes used by {@link BigQueryIO}.
+ */
+public class BigQueryHelpers {
+  @Nullable
+  /**
+   * Return a displayable string representation for a {@link TableReference}.
+   */
+  static ValueProvider<String> displayTable(
+      @Nullable ValueProvider<TableReference> table) {
+    if (table == null) {
+      return null;
+    }
+    return NestedValueProvider.of(table, new TableRefToTableSpec());
+  }
+
+  /**
+   * Returns a canonical string representation of the {@link TableReference}.
+   */
+  static String toTableSpec(TableReference ref) {
+    StringBuilder sb = new StringBuilder();
+    if (ref.getProjectId() != null) {
+      sb.append(ref.getProjectId());
+      sb.append(":");
+    }
+
+    sb.append(ref.getDatasetId()).append('.').append(ref.getTableId());
+    return sb.toString();
+  }
+
+  static <K, V> List<V> getOrCreateMapListValue(Map<K, List<V>> map, K key) {
+    List<V> value = map.get(key);
+    if (value == null) {
+      value = new ArrayList<>();
+      map.put(key, value);
+    }
+    return value;
+  }
+
+  /**
+   * Parse a table specification in the form
+   * {@code "[project_id]:[dataset_id].[table_id]"} or {@code "[dataset_id].[table_id]"}.
+   *
+   * <p>If the project id is omitted, the default project id is used.
+   */
+  static TableReference parseTableSpec(String tableSpec) {
+    Matcher match = BigQueryIO.TABLE_SPEC.matcher(tableSpec);
+    if (!match.matches()) {
+      throw new IllegalArgumentException(
+          "Table reference is not in [project_id]:[dataset_id].[table_id] "
+          + "format: " + tableSpec);
+    }
+
+    TableReference ref = new TableReference();
+    ref.setProjectId(match.group("PROJECT"));
+
+    return ref.setDatasetId(match.group("DATASET")).setTableId(match.group("TABLE"));
+  }
+
+  static String jobToPrettyString(@Nullable Job job) throws IOException {
+    return job == null ? "null" : job.toPrettyString();
+  }
+
+  static String statusToPrettyString(@Nullable JobStatus status) throws IOException {
+    return status == null ? "Unknown status: null." : status.toPrettyString();
+  }
+
+  static Status parseStatus(@Nullable Job job) {
+    if (job == null) {
+      return Status.UNKNOWN;
+    }
+    JobStatus status = job.getStatus();
+    if (status.getErrorResult() != null) {
+      return Status.FAILED;
+    } else if (status.getErrors() != null && !status.getErrors().isEmpty()) {
+      return Status.FAILED;
+    } else {
+      return Status.SUCCEEDED;
+    }
+  }
+
+  @VisibleForTesting
+  static String toJsonString(Object item) {
+    if (item == null) {
+      return null;
+    }
+    try {
+      return BigQueryIO.JSON_FACTORY.toString(item);
+    } catch (IOException e) {
+      throw new RuntimeException(
+          String.format("Cannot serialize %s to a JSON string.", item.getClass().getSimpleName()),
+          e);
+    }
+  }
+
+  @VisibleForTesting
+  static <T> T fromJsonString(String json, Class<T> clazz) {
+    if (json == null) {
+      return null;
+    }
+    try {
+      return BigQueryIO.JSON_FACTORY.fromString(json, clazz);
+    } catch (IOException e) {
+      throw new RuntimeException(
+          String.format("Cannot deserialize %s from a JSON string: %s.", clazz, json),
+          e);
+    }
+  }
+
+  /**
+   * Returns a randomUUID string.
+   *
+   * <p>{@code '-'} is removed because BigQuery doesn't allow it in dataset id.
+   */
+  static String randomUUIDString() {
+    return UUID.randomUUID().toString().replaceAll("-", "");
+  }
+
+  @VisibleForTesting
+  static class JsonSchemaToTableSchema
+      implements SerializableFunction<String, TableSchema> {
+    @Override
+    public TableSchema apply(String from) {
+      return fromJsonString(from, TableSchema.class);
+    }
+  }
+
+  @VisibleForTesting
+  static class BeamJobUuidToBigQueryJobUuid
+      implements SerializableFunction<String, String> {
+    @Override
+    public String apply(String from) {
+      return "beam_job_" + from;
+    }
+  }
+
+  static class TableSchemaToJsonSchema
+      implements SerializableFunction<TableSchema, String> {
+    @Override
+    public String apply(TableSchema from) {
+      return toJsonString(from);
+    }
+  }
+
+  static class JsonTableRefToTableRef
+      implements SerializableFunction<String, TableReference> {
+    @Override
+    public TableReference apply(String from) {
+      return fromJsonString(from, TableReference.class);
+    }
+  }
+
+  static class TableRefToTableSpec
+      implements SerializableFunction<TableReference, String> {
+    @Override
+    public String apply(TableReference from) {
+      return toTableSpec(from);
+    }
+  }
+
+  static class TableRefToJson
+      implements SerializableFunction<TableReference, String> {
+    @Override
+    public String apply(TableReference from) {
+      return toJsonString(from);
+    }
+  }
+
+  static class TableRefToProjectId
+      implements SerializableFunction<TableReference, String> {
+    @Override
+    public String apply(TableReference from) {
+      return from.getProjectId();
+    }
+  }
+
+  @VisibleForTesting
+  static class TableSpecToTableRef
+      implements SerializableFunction<String, TableReference> {
+    @Override
+    public TableReference apply(String from) {
+      return parseTableSpec(from);
+    }
+  }
+
+  @VisibleForTesting
+  static class CreatePerBeamJobUuid
+      implements SerializableFunction<String, String> {
+    private final String stepUuid;
+
+    CreatePerBeamJobUuid(String stepUuid) {
+      this.stepUuid = stepUuid;
+    }
+
+    @Override
+    public String apply(String jobUuid) {
+      return stepUuid + "_" + jobUuid.replaceAll("-", "");
+    }
+  }
+
+  @VisibleForTesting
+  static class CreateJsonTableRefFromUuid
+      implements SerializableFunction<String, TableReference> {
+    private final String executingProject;
+
+    public CreateJsonTableRefFromUuid(String executingProject) {
+      this.executingProject = executingProject;
+    }
+
+    @Override
+    public TableReference apply(String jobUuid) {
+      String queryTempDatasetId = "temp_dataset_" + jobUuid;
+      String queryTempTableId = "temp_table_" + jobUuid;
+      TableReference queryTempTableRef = new TableReference()
+          .setProjectId(executingProject)
+          .setDatasetId(queryTempDatasetId)
+          .setTableId(queryTempTableId);
+      return queryTempTableRef;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index d195afd..3f2f3e8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -25,13 +25,11 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobConfigurationExtract;
 import com.google.api.services.bigquery.model.JobConfigurationLoad;
 import com.google.api.services.bigquery.model.JobConfigurationQuery;
 import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
 import com.google.api.services.bigquery.model.JobReference;
 import com.google.api.services.bigquery.model.JobStatistics;
-import com.google.api.services.bigquery.model.JobStatus;
 import com.google.api.services.bigquery.model.Table;
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableRow;
@@ -39,38 +37,31 @@ import com.google.api.services.bigquery.model.TableSchema;
 import com.google.auto.value.AutoValue;
 import com.google.cloud.hadoop.util.ApiErrorExtractor;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.io.CountingOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.ObjectInputStream;
 import java.io.OutputStream;
-import java.io.Serializable;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
-import org.apache.avro.generic.GenericRecord;
+
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
@@ -82,8 +73,16 @@ import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.TableRowJsonCoder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.AvroSource;
 import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.BeamJobUuidToBigQueryJobUuid;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreateJsonTableRefFromUuid;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreatePerBeamJobUuid;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonTableRefToTableRef;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToTableSpec;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
 import org.apache.beam.sdk.metrics.Counter;
@@ -131,7 +130,6 @@ import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.sdk.values.ValueInSingleWindow;
-import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -153,7 +151,7 @@ import org.slf4j.LoggerFactory;
  * from the <a href="https://cloud.google.com/bigquery/client-libraries">
  * BigQuery Java Client API</a>.
  * Tables can be referred to as Strings, with or without the {@code projectId}.
- * A helper function is provided ({@link BigQueryIO#parseTableSpec(String)})
+ * A helper function is provided ({@link BigQueryHelpers#parseTableSpec(String)})
  * that parses the following string forms into a {@link TableReference}:
  *
  * <ul>
@@ -253,7 +251,7 @@ public class BigQueryIO {
    * Singleton instance of the JSON factory used to read and write JSON
    * formatted rows.
    */
-  private static final JsonFactory JSON_FACTORY = Transport.getJsonFactory();
+  static final JsonFactory JSON_FACTORY = Transport.getJsonFactory();
 
   /**
    * Project IDs must contain 6-63 lowercase letters, digits, or dashes.
@@ -281,7 +279,7 @@ public class BigQueryIO {
       String.format("((?<PROJECT>%s):)?(?<DATASET>%s)\\.(?<TABLE>%s)", PROJECT_ID_REGEXP,
           DATASET_REGEXP, TABLE_REGEXP);
 
-  private static final Pattern TABLE_SPEC = Pattern.compile(DATASET_TABLE_REGEXP);
+  static final Pattern TABLE_SPEC = Pattern.compile(DATASET_TABLE_REGEXP);
 
   private static final String RESOURCE_NOT_FOUND_ERROR =
       "BigQuery %1$s not found for table \"%2$s\" . Please create the %1$s before pipeline"
@@ -293,152 +291,6 @@ public class BigQueryIO {
           + " an earlier stage of the pipeline, this validation can be disabled using"
           + " #withoutValidation.";
 
-  /**
-   * Parse a table specification in the form
-   * {@code "[project_id]:[dataset_id].[table_id]"} or {@code "[dataset_id].[table_id]"}.
-   *
-   * <p>If the project id is omitted, the default project id is used.
-   */
-  public static TableReference parseTableSpec(String tableSpec) {
-    Matcher match = TABLE_SPEC.matcher(tableSpec);
-    if (!match.matches()) {
-      throw new IllegalArgumentException(
-          "Table reference is not in [project_id]:[dataset_id].[table_id] "
-          + "format: " + tableSpec);
-    }
-
-    TableReference ref = new TableReference();
-    ref.setProjectId(match.group("PROJECT"));
-
-    return ref.setDatasetId(match.group("DATASET")).setTableId(match.group("TABLE"));
-  }
-
-  /**
-   * Returns a canonical string representation of the {@link TableReference}.
-   */
-  public static String toTableSpec(TableReference ref) {
-    StringBuilder sb = new StringBuilder();
-    if (ref.getProjectId() != null) {
-      sb.append(ref.getProjectId());
-      sb.append(":");
-    }
-
-    sb.append(ref.getDatasetId()).append('.').append(ref.getTableId());
-    return sb.toString();
-  }
-
-  @VisibleForTesting
-  static class JsonSchemaToTableSchema
-      implements SerializableFunction<String, TableSchema> {
-    @Override
-    public TableSchema apply(String from) {
-      return fromJsonString(from, TableSchema.class);
-    }
-  }
-
-  private static class TableSchemaToJsonSchema
-      implements SerializableFunction<TableSchema, String> {
-    @Override
-    public String apply(TableSchema from) {
-      return toJsonString(from);
-    }
-  }
-
-  private static class JsonTableRefToTableRef
-      implements SerializableFunction<String, TableReference> {
-    @Override
-    public TableReference apply(String from) {
-      return fromJsonString(from, TableReference.class);
-    }
-  }
-
-  private static class TableRefToTableSpec
-      implements SerializableFunction<TableReference, String> {
-    @Override
-    public String apply(TableReference from) {
-      return toTableSpec(from);
-    }
-  }
-
-  private static class TableRefToJson
-      implements SerializableFunction<TableReference, String> {
-    @Override
-    public String apply(TableReference from) {
-      return toJsonString(from);
-    }
-  }
-
-  private static class TableRefToProjectId
-      implements SerializableFunction<TableReference, String> {
-    @Override
-    public String apply(TableReference from) {
-      return from.getProjectId();
-    }
-  }
-
-  @VisibleForTesting
-  static class TableSpecToTableRef
-      implements SerializableFunction<String, TableReference> {
-    @Override
-    public TableReference apply(String from) {
-      return parseTableSpec(from);
-    }
-  }
-
-  @VisibleForTesting
-  static class BeamJobUuidToBigQueryJobUuid
-      implements SerializableFunction<String, String> {
-    @Override
-    public String apply(String from) {
-      return "beam_job_" + from;
-    }
-  }
-
-  @VisibleForTesting
-  static class CreatePerBeamJobUuid
-      implements SerializableFunction<String, String> {
-    private final String stepUuid;
-
-    private CreatePerBeamJobUuid(String stepUuid) {
-      this.stepUuid = stepUuid;
-    }
-
-    @Override
-    public String apply(String jobUuid) {
-      return stepUuid + "_" + jobUuid.replaceAll("-", "");
-    }
-  }
-
-  @VisibleForTesting
-  static class CreateJsonTableRefFromUuid
-      implements SerializableFunction<String, TableReference> {
-    private final String executingProject;
-
-    private CreateJsonTableRefFromUuid(String executingProject) {
-      this.executingProject = executingProject;
-    }
-
-    @Override
-    public TableReference apply(String jobUuid) {
-      String queryTempDatasetId = "temp_dataset_" + jobUuid;
-      String queryTempTableId = "temp_table_" + jobUuid;
-      TableReference queryTempTableRef = new TableReference()
-          .setProjectId(executingProject)
-          .setDatasetId(queryTempDatasetId)
-          .setTableId(queryTempTableId);
-      return queryTempTableRef;
-    }
-  }
-
-  @Nullable
-  private static ValueProvider<String> displayTable(
-      @Nullable ValueProvider<TableReference> table) {
-    if (table == null) {
-      return null;
-    }
-    return NestedValueProvider.of(table, new TableRefToTableSpec());
-  }
-
 
   /**
    * A formatting function that maps a TableRow to itself. This allows sending a
@@ -556,7 +408,7 @@ public class BigQueryIO {
      * Read from table specified by a {@link TableReference}.
      */
     public Read from(TableReference table) {
-      return from(StaticValueProvider.of(toTableSpec(table)));
+      return from(StaticValueProvider.of(BigQueryHelpers.toTableSpec(table)));
     }
 
     private static final String QUERY_VALIDATION_FAILURE_ERROR =
@@ -672,7 +524,7 @@ public class BigQueryIO {
 
     @Override
     public PCollection<TableRow> expand(PBegin input) {
-      String stepUuid = randomUUIDString();
+      String stepUuid = BigQueryHelpers.randomUUIDString();
       BigQueryOptions bqOptions = input.getPipeline().getOptions().as(BigQueryOptions.class);
       ValueProvider<String> jobUuid = NestedValueProvider.of(
          StaticValueProvider.of(bqOptions.getJobName()), new CreatePerBeamJobUuid(stepUuid));
@@ -752,7 +604,7 @@ public class BigQueryIO {
     public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
       builder
-          .addIfNotNull(DisplayData.item("table", displayTable(getTableProvider()))
+          .addIfNotNull(DisplayData.item("table", BigQueryHelpers.displayTable(getTableProvider()))
             .withLabel("Table"))
           .addIfNotNull(DisplayData.item("query", getQuery())
             .withLabel("Query"))
@@ -787,7 +639,7 @@ public class BigQueryIO {
         TableReference tableRef = table.get();
         tableRef.setProjectId(bqOptions.getProject());
         return NestedValueProvider.of(StaticValueProvider.of(
-            toJsonString(tableRef)), new JsonTableRefToTableRef());
+            BigQueryHelpers.toJsonString(tableRef)), new JsonTableRefToTableRef());
       }
       return table;
     }
@@ -810,541 +662,15 @@ public class BigQueryIO {
     }
   }
 
-  /**
-   * A {@link PTransform} that invokes {@link CleanupOperation} after the input {@link PCollection}
-   * has been processed.
-   */
-  @VisibleForTesting
-  static class PassThroughThenCleanup<T> extends PTransform<PCollection<T>, PCollection<T>> {
-
-    private CleanupOperation cleanupOperation;
-
-    PassThroughThenCleanup(CleanupOperation cleanupOperation) {
-      this.cleanupOperation = cleanupOperation;
-    }
-
-    @Override
-    public PCollection<T> expand(PCollection<T> input) {
-      TupleTag<T> mainOutput = new TupleTag<>();
-      TupleTag<Void> cleanupSignal = new TupleTag<>();
-      PCollectionTuple outputs = input.apply(ParDo.of(new IdentityFn<T>())
-          .withOutputTags(mainOutput, TupleTagList.of(cleanupSignal)));
-
-      PCollectionView<Void> cleanupSignalView = outputs.get(cleanupSignal)
-          .setCoder(VoidCoder.of())
-          .apply(View.<Void>asSingleton().withDefaultValue(null));
-
-      input.getPipeline()
-          .apply("Create(CleanupOperation)", Create.of(cleanupOperation))
-          .apply("Cleanup", ParDo.of(
-              new DoFn<CleanupOperation, Void>() {
-                @ProcessElement
-                public void processElement(ProcessContext c)
-                    throws Exception {
-                  c.element().cleanup(c.getPipelineOptions());
-                }
-              }).withSideInputs(cleanupSignalView));
-
-      return outputs.get(mainOutput);
-    }
-
-    private static class IdentityFn<T> extends DoFn<T, T> {
-      @ProcessElement
-      public void processElement(ProcessContext c) {
-        c.output(c.element());
-      }
-    }
-
-    abstract static class CleanupOperation implements Serializable {
-      abstract void cleanup(PipelineOptions options) throws Exception;
-    }
-  }
-
-  /**
-   * A {@link BigQuerySourceBase} for reading BigQuery tables.
-   */
-  @VisibleForTesting
-  static class BigQueryTableSource extends BigQuerySourceBase {
-
-    static BigQueryTableSource create(
-        ValueProvider<String> jobIdToken,
-        ValueProvider<TableReference> table,
-        String extractDestinationDir,
-        BigQueryServices bqServices,
-        ValueProvider<String> executingProject) {
-      return new BigQueryTableSource(
-          jobIdToken, table, extractDestinationDir, bqServices, executingProject);
-    }
-
-    private final ValueProvider<String> jsonTable;
-    private final AtomicReference<Long> tableSizeBytes;
-
-    private BigQueryTableSource(
-        ValueProvider<String> jobIdToken,
-        ValueProvider<TableReference> table,
-        String extractDestinationDir,
-        BigQueryServices bqServices,
-        ValueProvider<String> executingProject) {
-      super(jobIdToken, extractDestinationDir, bqServices, executingProject);
-      this.jsonTable = NestedValueProvider.of(checkNotNull(table, "table"), new TableRefToJson());
-      this.tableSizeBytes = new AtomicReference<>();
-    }
-
-    @Override
-    protected TableReference getTableToExtract(BigQueryOptions bqOptions) throws IOException {
-      checkState(jsonTable.isAccessible());
-      return JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
-    }
-
-    @Override
-    public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
-      BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
-      checkState(jsonTable.isAccessible());
-      TableReference tableRef = JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
-      return new BigQueryReader(this, bqServices.getReaderFromTable(bqOptions, tableRef));
-    }
-
-    @Override
-    public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
-      if (tableSizeBytes.get() == null) {
-        TableReference table = JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
-
-        Long numBytes = bqServices.getDatasetService(options.as(BigQueryOptions.class))
-            .getTable(table).getNumBytes();
-        tableSizeBytes.compareAndSet(null, numBytes);
-      }
-      return tableSizeBytes.get();
-    }
-
-    @Override
-    protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
-      // Do nothing.
-    }
-
-    @Override
-    public void populateDisplayData(DisplayData.Builder builder) {
-      super.populateDisplayData(builder);
-      builder.add(DisplayData.item("table", jsonTable));
-    }
-  }
-
-  /**
-   * A {@link BigQuerySourceBase} for querying BigQuery tables.
-   */
-  @VisibleForTesting
-  static class BigQueryQuerySource extends BigQuerySourceBase {
-
-    static BigQueryQuerySource create(
-        ValueProvider<String> jobIdToken,
-        ValueProvider<String> query,
-        ValueProvider<TableReference> queryTempTableRef,
-        Boolean flattenResults,
-        Boolean useLegacySql,
-        String extractDestinationDir,
-        BigQueryServices bqServices) {
-      return new BigQueryQuerySource(
-          jobIdToken,
-          query,
-          queryTempTableRef,
-          flattenResults,
-          useLegacySql,
-          extractDestinationDir,
-          bqServices);
-    }
-
-    private final ValueProvider<String> query;
-    private final ValueProvider<String> jsonQueryTempTable;
-    private final Boolean flattenResults;
-    private final Boolean useLegacySql;
-    private transient AtomicReference<JobStatistics> dryRunJobStats;
-
-    private BigQueryQuerySource(
-        ValueProvider<String> jobIdToken,
-        ValueProvider<String> query,
-        ValueProvider<TableReference> queryTempTableRef,
-        Boolean flattenResults,
-        Boolean useLegacySql,
-        String extractDestinationDir,
-        BigQueryServices bqServices) {
-      super(jobIdToken, extractDestinationDir, bqServices,
-          NestedValueProvider.of(
-              checkNotNull(queryTempTableRef, "queryTempTableRef"), new TableRefToProjectId()));
-      this.query = checkNotNull(query, "query");
-      this.jsonQueryTempTable = NestedValueProvider.of(
-          queryTempTableRef, new TableRefToJson());
-      this.flattenResults = checkNotNull(flattenResults, "flattenResults");
-      this.useLegacySql = checkNotNull(useLegacySql, "useLegacySql");
-      this.dryRunJobStats = new AtomicReference<>();
-    }
-
-    @Override
-    public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
-      BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
-      return dryRunQueryIfNeeded(bqOptions).getTotalBytesProcessed();
-    }
-
-    @Override
-    public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
-      BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
-      return new BigQueryReader(this, bqServices.getReaderFromQuery(
-          bqOptions, executingProject.get(), createBasicQueryConfig()));
-    }
-
-    @Override
-    protected TableReference getTableToExtract(BigQueryOptions bqOptions)
-        throws IOException, InterruptedException {
-      // 1. Find the location of the query.
-      String location = null;
-      List<TableReference> referencedTables =
-          dryRunQueryIfNeeded(bqOptions).getQuery().getReferencedTables();
-      DatasetService tableService = bqServices.getDatasetService(bqOptions);
-      if (referencedTables != null && !referencedTables.isEmpty()) {
-        TableReference queryTable = referencedTables.get(0);
-        location = tableService.getTable(queryTable).getLocation();
-      }
-
-      // 2. Create the temporary dataset in the query location.
-      TableReference tableToExtract =
-          JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
-      tableService.createDataset(
-          tableToExtract.getProjectId(),
-          tableToExtract.getDatasetId(),
-          location,
-          "Dataset for BigQuery query job temporary table");
-
-      // 3. Execute the query.
-      String queryJobId = jobIdToken.get() + "-query";
-      executeQuery(
-          executingProject.get(),
-          queryJobId,
-          tableToExtract,
-          bqServices.getJobService(bqOptions));
-      return tableToExtract;
-    }
-
-    @Override
-    protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
-      checkState(jsonQueryTempTable.isAccessible());
-      TableReference tableToRemove =
-          JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
-
-      DatasetService tableService = bqServices.getDatasetService(bqOptions);
-      tableService.deleteTable(tableToRemove);
-      tableService.deleteDataset(tableToRemove.getProjectId(), tableToRemove.getDatasetId());
-    }
-
-    @Override
-    public void populateDisplayData(DisplayData.Builder builder) {
-      super.populateDisplayData(builder);
-      builder.add(DisplayData.item("query", query));
-    }
-
-    private synchronized JobStatistics dryRunQueryIfNeeded(BigQueryOptions bqOptions)
-        throws InterruptedException, IOException {
-      if (dryRunJobStats.get() == null) {
-        JobStatistics jobStats = bqServices.getJobService(bqOptions).dryRunQuery(
-            executingProject.get(), createBasicQueryConfig());
-        dryRunJobStats.compareAndSet(null, jobStats);
-      }
-      return dryRunJobStats.get();
-    }
-
-    private void executeQuery(
-        String executingProject,
-        String jobId,
-        TableReference destinationTable,
-        JobService jobService) throws IOException, InterruptedException {
-      JobReference jobRef = new JobReference()
-          .setProjectId(executingProject)
-          .setJobId(jobId);
-
-      JobConfigurationQuery queryConfig = createBasicQueryConfig()
-          .setAllowLargeResults(true)
-          .setCreateDisposition("CREATE_IF_NEEDED")
-          .setDestinationTable(destinationTable)
-          .setPriority("BATCH")
-          .setWriteDisposition("WRITE_EMPTY");
-
-      jobService.startQueryJob(jobRef, queryConfig);
-      Job job = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
-      if (parseStatus(job) != Status.SUCCEEDED) {
-        throw new IOException(String.format(
-            "Query job %s failed, status: %s.", jobId, statusToPrettyString(job.getStatus())));
-      }
-    }
-
-    private JobConfigurationQuery createBasicQueryConfig() {
-      return new JobConfigurationQuery()
-          .setFlattenResults(flattenResults)
-          .setQuery(query.get())
-          .setUseLegacySql(useLegacySql);
-    }
-
-    private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException {
-      in.defaultReadObject();
-      dryRunJobStats = new AtomicReference<>();
-    }
-  }
-
-  /**
-   * An abstract {@link BoundedSource} to read a table from BigQuery.
-   *
-   * <p>This source uses a BigQuery export job to take a snapshot of the table on GCS, and then
-   * reads in parallel from each produced file. It is implemented by {@link BigQueryTableSource},
-   * and {@link BigQueryQuerySource}, depending on the configuration of the read.
-   * Specifically,
-   * <ul>
-   * <li>{@link BigQueryTableSource} is for reading BigQuery tables</li>
-   * <li>{@link BigQueryQuerySource} is for querying BigQuery tables</li>
-   * </ul>
-   * ...
-   */
-  private abstract static class BigQuerySourceBase extends BoundedSource<TableRow> {
-    // The maximum number of retries to poll a BigQuery job.
-    protected static final int JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
-
-    protected final ValueProvider<String> jobIdToken;
-    protected final String extractDestinationDir;
-    protected final BigQueryServices bqServices;
-    protected final ValueProvider<String> executingProject;
-
-    private BigQuerySourceBase(
-        ValueProvider<String> jobIdToken,
-        String extractDestinationDir,
-        BigQueryServices bqServices,
-        ValueProvider<String> executingProject) {
-      this.jobIdToken = checkNotNull(jobIdToken, "jobIdToken");
-      this.extractDestinationDir = checkNotNull(extractDestinationDir, "extractDestinationDir");
-      this.bqServices = checkNotNull(bqServices, "bqServices");
-      this.executingProject = checkNotNull(executingProject, "executingProject");
-    }
-
-    @Override
-    public List<BoundedSource<TableRow>> splitIntoBundles(
-        long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
-      BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
-      TableReference tableToExtract = getTableToExtract(bqOptions);
-      JobService jobService = bqServices.getJobService(bqOptions);
-      String extractJobId = getExtractJobId(jobIdToken);
-      List<String> tempFiles = executeExtract(extractJobId, tableToExtract, jobService);
-
-      TableSchema tableSchema = bqServices.getDatasetService(bqOptions)
-          .getTable(tableToExtract).getSchema();
-
-      cleanupTempResource(bqOptions);
-      return createSources(tempFiles, tableSchema);
-    }
-
-    protected abstract TableReference getTableToExtract(BigQueryOptions bqOptions) throws Exception;
-
-    protected abstract void cleanupTempResource(BigQueryOptions bqOptions) throws Exception;
-
-    @Override
-    public void validate() {
-      // Do nothing, validation is done in BigQuery.Read.
-    }
-
-    @Override
-    public Coder<TableRow> getDefaultOutputCoder() {
-      return TableRowJsonCoder.of();
-    }
-
-    private List<String> executeExtract(
-        String jobId, TableReference table, JobService jobService)
-            throws InterruptedException, IOException {
-      JobReference jobRef = new JobReference()
-          .setProjectId(executingProject.get())
-          .setJobId(jobId);
-
-      String destinationUri = getExtractDestinationUri(extractDestinationDir);
-      JobConfigurationExtract extract = new JobConfigurationExtract()
-          .setSourceTable(table)
-          .setDestinationFormat("AVRO")
-          .setDestinationUris(ImmutableList.of(destinationUri));
-
-      LOG.info("Starting BigQuery extract job: {}", jobId);
-      jobService.startExtractJob(jobRef, extract);
-      Job extractJob =
-          jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
-      if (parseStatus(extractJob) != Status.SUCCEEDED) {
-        throw new IOException(String.format(
-            "Extract job %s failed, status: %s.",
-            extractJob.getJobReference().getJobId(), statusToPrettyString(extractJob.getStatus())));
-      }
-
-      List<String> tempFiles = getExtractFilePaths(extractDestinationDir, extractJob);
-      return ImmutableList.copyOf(tempFiles);
-    }
-
-    private List<BoundedSource<TableRow>> createSources(
-        List<String> files, TableSchema tableSchema) throws IOException, InterruptedException {
-      final String jsonSchema = JSON_FACTORY.toString(tableSchema);
-
-      SerializableFunction<GenericRecord, TableRow> function =
-          new SerializableFunction<GenericRecord, TableRow>() {
-            @Override
-            public TableRow apply(GenericRecord input) {
-              return BigQueryAvroUtils.convertGenericRecordToTableRow(
-                  input, fromJsonString(jsonSchema, TableSchema.class));
-            }};
-
-      List<BoundedSource<TableRow>> avroSources = Lists.newArrayList();
-      for (String fileName : files) {
-        avroSources.add(new TransformingSource<>(
-            AvroSource.from(fileName), function, getDefaultOutputCoder()));
-      }
-      return ImmutableList.copyOf(avroSources);
-    }
-
-    protected static class BigQueryReader extends BoundedSource.BoundedReader<TableRow> {
-      private final BigQuerySourceBase source;
-      private final BigQueryServices.BigQueryJsonReader reader;
-
-      private BigQueryReader(
-          BigQuerySourceBase source, BigQueryServices.BigQueryJsonReader reader) {
-        this.source = source;
-        this.reader = reader;
-      }
-
-      @Override
-      public BoundedSource<TableRow> getCurrentSource() {
-        return source;
-      }
-
-      @Override
-      public boolean start() throws IOException {
-        return reader.start();
-      }
-
-      @Override
-      public boolean advance() throws IOException {
-        return reader.advance();
-      }
-
-      @Override
-      public TableRow getCurrent() throws NoSuchElementException {
-        return reader.getCurrent();
-      }
-
-      @Override
-      public void close() throws IOException {
-        reader.close();
-      }
-    }
-  }
-
-  /**
-   * A {@link BoundedSource} that reads from {@code BoundedSource<T>}
-   * and transforms elements to type {@code V}.
-  */
-  @VisibleForTesting
-  static class TransformingSource<T, V> extends BoundedSource<V> {
-    private final BoundedSource<T> boundedSource;
-    private final SerializableFunction<T, V> function;
-    private final Coder<V> outputCoder;
-
-    TransformingSource(
-        BoundedSource<T> boundedSource,
-        SerializableFunction<T, V> function,
-        Coder<V> outputCoder) {
-      this.boundedSource = checkNotNull(boundedSource, "boundedSource");
-      this.function = checkNotNull(function, "function");
-      this.outputCoder = checkNotNull(outputCoder, "outputCoder");
-    }
-
-    @Override
-    public List<? extends BoundedSource<V>> splitIntoBundles(
-        long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
-      return Lists.transform(
-          boundedSource.splitIntoBundles(desiredBundleSizeBytes, options),
-          new Function<BoundedSource<T>, BoundedSource<V>>() {
-            @Override
-            public BoundedSource<V> apply(BoundedSource<T> input) {
-              return new TransformingSource<>(input, function, outputCoder);
-            }
-          });
-    }
-
-    @Override
-    public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
-      return boundedSource.getEstimatedSizeBytes(options);
-    }
-
-    @Override
-    public BoundedReader<V> createReader(PipelineOptions options) throws IOException {
-      return new TransformingReader(boundedSource.createReader(options));
-    }
-
-    @Override
-    public void validate() {
-      boundedSource.validate();
-    }
-
-    @Override
-    public Coder<V> getDefaultOutputCoder() {
-      return outputCoder;
-    }
-
-    private class TransformingReader extends BoundedReader<V> {
-      private final BoundedReader<T> boundedReader;
-
-      private TransformingReader(BoundedReader<T> boundedReader) {
-        this.boundedReader = checkNotNull(boundedReader, "boundedReader");
-      }
-
-      @Override
-      public synchronized BoundedSource<V> getCurrentSource() {
-        return new TransformingSource<>(boundedReader.getCurrentSource(), function, outputCoder);
-      }
-
-      @Override
-      public boolean start() throws IOException {
-        return boundedReader.start();
-      }
-
-      @Override
-      public boolean advance() throws IOException {
-        return boundedReader.advance();
-      }
-
-      @Override
-      public V getCurrent() throws NoSuchElementException {
-        T current = boundedReader.getCurrent();
-        return function.apply(current);
-      }
-
-      @Override
-      public void close() throws IOException {
-        boundedReader.close();
-      }
-
-      @Override
-      public synchronized BoundedSource<V> splitAtFraction(double fraction) {
-        BoundedSource<T> split = boundedReader.splitAtFraction(fraction);
-        return split == null ? null : new TransformingSource<>(split, function, outputCoder);
-      }
-
-      @Override
-      public Double getFractionConsumed() {
-        return boundedReader.getFractionConsumed();
-      }
-
-      @Override
-      public Instant getCurrentTimestamp() throws NoSuchElementException {
-        return boundedReader.getCurrentTimestamp();
-      }
-    }
-  }
-
-  private static String getExtractJobId(ValueProvider<String> jobIdToken) {
+  static String getExtractJobId(ValueProvider<String> jobIdToken) {
     return jobIdToken.get() + "-extract";
   }
 
-  private static String getExtractDestinationUri(String extractDestinationDir) {
+  static String getExtractDestinationUri(String extractDestinationDir) {
     return String.format("%s/%s", extractDestinationDir, "*.avro");
   }
 
-  private static List<String> getExtractFilePaths(String extractDestinationDir, Job extractJob)
+  static List<String> getExtractFilePaths(String extractDestinationDir, Job extractJob)
       throws IOException {
     JobStatistics jobStats = extractJob.getStatistics();
     List<Long> counts = jobStats.getExtract().getDestinationUriFileCounts();
@@ -1538,7 +864,8 @@ public class BigQueryIO {
     }
 
     /**
-     * Writes to the given table, specified in the format described in {@link #parseTableSpec}.
+     * Writes to the given table, specified in the format described in
+     * {@link BigQueryHelpers#parseTableSpec}.
      */
     public Write<T> to(String tableSpec) {
       return to(StaticValueProvider.of(tableSpec));
@@ -1546,7 +873,7 @@ public class BigQueryIO {
 
     /** Writes to the given table, specified as a {@link TableReference}. */
     public Write<T> to(TableReference table) {
-      return to(StaticValueProvider.of(toTableSpec(table)));
+      return to(StaticValueProvider.of(BigQueryHelpers.toTableSpec(table)));
     }
 
     /** Same as {@link #to(String)}, but with a {@link ValueProvider}. */
@@ -1596,7 +923,7 @@ public class BigQueryIO {
 
       @Override
       public TableReference apply(ValueInSingleWindow<T> value) {
-        return parseTableSpec(tableSpecFunction.apply(value));
+        return BigQueryHelpers.parseTableSpec(tableSpecFunction.apply(value));
       }
     }
 
@@ -1609,7 +936,7 @@ public class BigQueryIO {
      */
     public Write<T> withSchema(TableSchema schema) {
       return toBuilder()
-          .setJsonSchema(StaticValueProvider.of(toJsonString(schema)))
+          .setJsonSchema(StaticValueProvider.of(BigQueryHelpers.toJsonString(schema)))
           .build();
     }
 
@@ -1655,7 +982,7 @@ public class BigQueryIO {
           checkState(
               datasetService.isTableEmpty(tableRef),
               "BigQuery table is not empty: %s.",
-              BigQueryIO.toTableSpec(tableRef));
+              BigQueryHelpers.toTableSpec(tableRef));
         }
       } catch (IOException | InterruptedException e) {
         if (e instanceof InterruptedException) {
@@ -1663,7 +990,7 @@ public class BigQueryIO {
         }
         throw new RuntimeException(
             "unable to confirm BigQuery table emptiness for table "
-                + BigQueryIO.toTableSpec(tableRef), e);
+                + BigQueryHelpers.toTableSpec(tableRef), e);
       }
     }
 
@@ -1758,7 +1085,7 @@ public class BigQueryIO {
 
       ValueProvider<TableReference> table = getTableWithDefaultProject(options);
 
-      final String stepUuid = randomUUIDString();
+      final String stepUuid = BigQueryHelpers.randomUUIDString();
 
       String tempLocation = options.getTempLocation();
       String tempFilePrefix;
@@ -1953,7 +1280,7 @@ public class BigQueryIO {
 
     /** Returns the table schema. */
     public TableSchema getSchema() {
-      return fromJsonString(
+      return BigQueryHelpers.fromJsonString(
           getJsonSchema() == null ? null : getJsonSchema().get(), TableSchema.class);
     }
 
@@ -1979,7 +1306,7 @@ public class BigQueryIO {
         TableReference tableRef = table.get();
         tableRef.setProjectId(bqOptions.getProject());
         return NestedValueProvider.of(StaticValueProvider.of(
-            toJsonString(tableRef)), new JsonTableRefToTableRef());
+            BigQueryHelpers.toJsonString(tableRef)), new JsonTableRefToTableRef());
       }
       return table;
     }
@@ -2128,7 +1455,8 @@ public class BigQueryIO {
         List<String> partition = Lists.newArrayList(c.element().getValue()).get(0);
         String jobIdPrefix = String.format(
             c.sideInput(jobIdToken) + "_%05d", c.element().getKey());
-        TableReference ref = fromJsonString(jsonTableRef.get(), TableReference.class);
+        TableReference ref = BigQueryHelpers.fromJsonString(jsonTableRef.get(),
+            TableReference.class);
         if (!singlePartition) {
           ref.setTableId(jobIdPrefix);
         }
@@ -2138,13 +1466,13 @@ public class BigQueryIO {
             bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
             jobIdPrefix,
             ref,
-            fromJsonString(
+            BigQueryHelpers.fromJsonString(
                 jsonSchema == null ? null : jsonSchema.get(), TableSchema.class),
             partition,
             writeDisposition,
             createDisposition,
             tableDescription);
-        c.output(toJsonString(ref));
+        c.output(BigQueryHelpers.toJsonString(ref));
 
         removeTemporaryFiles(c.getPipelineOptions(), tempFilePrefix, partition);
       }
@@ -2176,7 +1504,7 @@ public class BigQueryIO {
               .setJobId(jobId);
           jobService.startLoadJob(jobRef, loadConfig);
           Job loadJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
-          Status jobStatus = parseStatus(loadJob);
+          Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
           switch (jobStatus) {
             case SUCCEEDED:
               if (tableDescription != null) {
@@ -2185,14 +1513,15 @@ public class BigQueryIO {
               return;
             case UNKNOWN:
               throw new RuntimeException(String.format(
-                  "UNKNOWN status of load job [%s]: %s.", jobId, jobToPrettyString(loadJob)));
+                  "UNKNOWN status of load job [%s]: %s.", jobId,
+                  BigQueryHelpers.jobToPrettyString(loadJob)));
             case FAILED:
               lastFailedLoadJob = loadJob;
               continue;
             default:
               throw new IllegalStateException(String.format(
                   "Unexpected status [%s] of load job: %s.",
-                  jobStatus, jobToPrettyString(loadJob)));
+                  jobStatus, BigQueryHelpers.jobToPrettyString(loadJob)));
           }
         }
         throw new RuntimeException(String.format(
@@ -2200,7 +1529,7 @@ public class BigQueryIO {
                 + "reached max retries: %d, last failed load job: %s.",
             jobIdPrefix,
             Write.MAX_RETRY_JOBS,
-            jobToPrettyString(lastFailedLoadJob)));
+            BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
       }
 
       static void removeTemporaryFiles(
@@ -2281,13 +1610,13 @@ public class BigQueryIO {
 
         List<TableReference> tempTables = Lists.newArrayList();
         for (String table : tempTablesJson) {
-          tempTables.add(fromJsonString(table, TableReference.class));
+          tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
         }
         copy(
             bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
             bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
             c.sideInput(jobIdToken),
-            fromJsonString(jsonTableRef.get(), TableReference.class),
+            BigQueryHelpers.fromJsonString(jsonTableRef.get(), TableReference.class),
             tempTables,
             writeDisposition,
             createDisposition,
@@ -2322,7 +1651,7 @@ public class BigQueryIO {
               .setJobId(jobId);
           jobService.startCopyJob(jobRef, copyConfig);
           Job copyJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
-          Status jobStatus = parseStatus(copyJob);
+          Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
           switch (jobStatus) {
             case SUCCEEDED:
               if (tableDescription != null) {
@@ -2331,14 +1660,15 @@ public class BigQueryIO {
               return;
             case UNKNOWN:
               throw new RuntimeException(String.format(
-                  "UNKNOWN status of copy job [%s]: %s.", jobId, jobToPrettyString(copyJob)));
+                  "UNKNOWN status of copy job [%s]: %s.", jobId,
+                  BigQueryHelpers.jobToPrettyString(copyJob)));
             case FAILED:
               lastFailedCopyJob = copyJob;
               continue;
             default:
               throw new IllegalStateException(String.format(
                   "Unexpected status [%s] of load job: %s.",
-                  jobStatus, jobToPrettyString(copyJob)));
+                  jobStatus, BigQueryHelpers.jobToPrettyString(copyJob)));
           }
         }
         throw new RuntimeException(String.format(
@@ -2346,17 +1676,17 @@ public class BigQueryIO {
                 + "reached max retries: %d, last failed copy job: %s.",
             jobIdPrefix,
             Write.MAX_RETRY_JOBS,
-            jobToPrettyString(lastFailedCopyJob)));
+            BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
       }
 
       static void removeTemporaryTables(DatasetService tableService,
           List<TableReference> tempTables) {
         for (TableReference tableRef : tempTables) {
           try {
-            LOG.debug("Deleting table {}", toJsonString(tableRef));
+            LOG.debug("Deleting table {}", BigQueryHelpers.toJsonString(tableRef));
             tableService.deleteTable(tableRef);
           } catch (Exception e) {
-            LOG.warn("Failed to delete the table {}", toJsonString(tableRef), e);
+            LOG.warn("Failed to delete the table {}", BigQueryHelpers.toJsonString(tableRef), e);
           }
         }
       }
@@ -2376,14 +1706,6 @@ public class BigQueryIO {
     }
   }
 
-  private static String jobToPrettyString(@Nullable Job job) throws IOException {
-    return job == null ? "null" : job.toPrettyString();
-  }
-
-  private static String statusToPrettyString(@Nullable JobStatus status) throws IOException {
-    return status == null ? "Unknown status: null." : status.toPrettyString();
-  }
-
   private static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
     try {
       datasetService.getDataset(table.getProjectId(), table.getDatasetId());
@@ -2391,14 +1713,14 @@ public class BigQueryIO {
       ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
       if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
         throw new IllegalArgumentException(
-            String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", BigQueryIO.toTableSpec(table)),
+            String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", BigQueryHelpers.toTableSpec(table)),
             e);
       } else if (e instanceof  RuntimeException) {
         throw (RuntimeException) e;
       } else {
         throw new RuntimeException(
             String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "dataset",
-                BigQueryIO.toTableSpec(table)),
+                BigQueryHelpers.toTableSpec(table)),
             e);
       }
     }
@@ -2411,13 +1733,14 @@ public class BigQueryIO {
       ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
       if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
         throw new IllegalArgumentException(
-            String.format(RESOURCE_NOT_FOUND_ERROR, "table", BigQueryIO.toTableSpec(table)), e);
+            String.format(RESOURCE_NOT_FOUND_ERROR, "table", BigQueryHelpers.toTableSpec(table)),
+            e);
       } else if (e instanceof  RuntimeException) {
         throw (RuntimeException) e;
       } else {
         throw new RuntimeException(
             String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "table",
-                BigQueryIO.toTableSpec(table)),
+                BigQueryHelpers.toTableSpec(table)),
             e);
       }
     }
@@ -2492,8 +1815,9 @@ public class BigQueryIO {
     @ProcessElement
     public void processElement(ProcessContext context) {
       String tableSpec = context.element().getKey().getKey();
-      List<TableRow> rows = getOrCreateMapListValue(tableRows, tableSpec);
-      List<String> uniqueIds = getOrCreateMapListValue(uniqueIdsForTableRows, tableSpec);
+      List<TableRow> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
+      List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows,
+          tableSpec);
 
       rows.add(context.element().getValue().tableRow);
       uniqueIds.add(context.element().getValue().uniqueId);
@@ -2526,7 +1850,7 @@ public class BigQueryIO {
 
     public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
         throws InterruptedException, IOException {
-      TableReference tableReference = parseTableSpec(tableSpec);
+      TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec);
       if (createDisposition != createDisposition.CREATE_NEVER
           && !createdTables.contains(tableSpec)) {
         synchronized (createdTables) {
@@ -2723,7 +2047,7 @@ public class BigQueryIO {
           TableReference tableRef = table.get()
               .setProjectId(options.as(BigQueryOptions.class).getProject());
           table = NestedValueProvider.of(
-              StaticValueProvider.of(toJsonString(tableRef)),
+              StaticValueProvider.of(BigQueryHelpers.toJsonString(tableRef)),
               new JsonTableRefToTableRef());
         }
         this.tableSpec = NestedValueProvider.of(table, new TableRefToTableSpec());
@@ -2779,7 +2103,7 @@ public class BigQueryIO {
         if (table.getProjectId() == null) {
           table.setProjectId(options.getProject());
         }
-        return toTableSpec(table);
+        return BigQueryHelpers.toTableSpec(table);
       }
     }
   }
@@ -2858,68 +2182,9 @@ public class BigQueryIO {
     UNKNOWN,
   }
 
-  private static Status parseStatus(@Nullable Job job) {
-    if (job == null) {
-      return Status.UNKNOWN;
-    }
-    JobStatus status = job.getStatus();
-    if (status.getErrorResult() != null) {
-      return Status.FAILED;
-    } else if (status.getErrors() != null && !status.getErrors().isEmpty()) {
-      return Status.FAILED;
-    } else {
-      return Status.SUCCEEDED;
-    }
-  }
-
-  @VisibleForTesting
-  static String toJsonString(Object item) {
-    if (item == null) {
-      return null;
-    }
-    try {
-      return JSON_FACTORY.toString(item);
-    } catch (IOException e) {
-      throw new RuntimeException(
-          String.format("Cannot serialize %s to a JSON string.", item.getClass().getSimpleName()),
-          e);
-    }
-  }
-
-  @VisibleForTesting
-  static <T> T fromJsonString(String json, Class<T> clazz) {
-    if (json == null) {
-      return null;
-    }
-    try {
-      return JSON_FACTORY.fromString(json, clazz);
-    } catch (IOException e) {
-      throw new RuntimeException(
-          String.format("Cannot deserialize %s from a JSON string: %s.", clazz, json),
-          e);
-    }
-  }
-
-  /**
-   * Returns a randomUUID string.
-   *
-   * <p>{@code '-'} is removed because BigQuery doesn't allow it in dataset id.
-   */
-  private static String randomUUIDString() {
-    return UUID.randomUUID().toString().replaceAll("-", "");
-  }
-
   /////////////////////////////////////////////////////////////////////////////
 
   /** Disallow construction of utility class. */
   private BigQueryIO() {}
 
-  private static <K, V> List<V> getOrCreateMapListValue(Map<K, List<V>> map, K key) {
-    List<V> value = map.get(key);
-    if (value == null) {
-      value = new ArrayList<>();
-      map.put(key, value);
-    }
-    return value;
-  }
 }

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
new file mode 100644
index 0000000..a909957
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
@@ -0,0 +1,186 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationQuery;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.JobStatistics;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToProjectId;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+
+/**
+ * A {@link BigQuerySourceBase} for querying BigQuery tables.
+ */
+@VisibleForTesting
+class BigQueryQuerySource extends BigQuerySourceBase {
+
+  static BigQueryQuerySource create(
+      ValueProvider<String> jobIdToken,
+      ValueProvider<String> query,
+      ValueProvider<TableReference> queryTempTableRef,
+      Boolean flattenResults,
+      Boolean useLegacySql,
+      String extractDestinationDir,
+      BigQueryServices bqServices) {
+    return new BigQueryQuerySource(
+        jobIdToken,
+        query,
+        queryTempTableRef,
+        flattenResults,
+        useLegacySql,
+        extractDestinationDir,
+        bqServices);
+  }
+
+  private final ValueProvider<String> query;
+  private final ValueProvider<String> jsonQueryTempTable;
+  private final Boolean flattenResults;
+  private final Boolean useLegacySql;
+  private transient AtomicReference<JobStatistics> dryRunJobStats;
+
+  private BigQueryQuerySource(
+      ValueProvider<String> jobIdToken,
+      ValueProvider<String> query,
+      ValueProvider<TableReference> queryTempTableRef,
+      Boolean flattenResults,
+      Boolean useLegacySql,
+      String extractDestinationDir,
+      BigQueryServices bqServices) {
+    super(jobIdToken, extractDestinationDir, bqServices,
+        NestedValueProvider.of(
+            checkNotNull(queryTempTableRef, "queryTempTableRef"), new TableRefToProjectId()));
+    this.query = checkNotNull(query, "query");
+    this.jsonQueryTempTable = NestedValueProvider.of(
+        queryTempTableRef, new TableRefToJson());
+    this.flattenResults = checkNotNull(flattenResults, "flattenResults");
+    this.useLegacySql = checkNotNull(useLegacySql, "useLegacySql");
+    this.dryRunJobStats = new AtomicReference<>();
+  }
+
+  @Override
+  public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+    BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+    return dryRunQueryIfNeeded(bqOptions).getTotalBytesProcessed();
+  }
+
+  @Override
+  public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
+    BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+    return new BigQueryReader(this, bqServices.getReaderFromQuery(
+        bqOptions, executingProject.get(), createBasicQueryConfig()));
+  }
+
+  @Override
+  protected TableReference getTableToExtract(BigQueryOptions bqOptions)
+      throws IOException, InterruptedException {
+    // 1. Find the location of the query.
+    String location = null;
+    List<TableReference> referencedTables =
+        dryRunQueryIfNeeded(bqOptions).getQuery().getReferencedTables();
+    DatasetService tableService = bqServices.getDatasetService(bqOptions);
+    if (referencedTables != null && !referencedTables.isEmpty()) {
+      TableReference queryTable = referencedTables.get(0);
+      location = tableService.getTable(queryTable).getLocation();
+    }
+
+    // 2. Create the temporary dataset in the query location.
+    TableReference tableToExtract =
+        BigQueryIO.JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
+    tableService.createDataset(
+        tableToExtract.getProjectId(),
+        tableToExtract.getDatasetId(),
+        location,
+        "Dataset for BigQuery query job temporary table");
+
+    // 3. Execute the query.
+    String queryJobId = jobIdToken.get() + "-query";
+    executeQuery(
+        executingProject.get(),
+        queryJobId,
+        tableToExtract,
+        bqServices.getJobService(bqOptions));
+    return tableToExtract;
+  }
+
+  @Override
+  protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
+    checkState(jsonQueryTempTable.isAccessible());
+    TableReference tableToRemove =
+        BigQueryIO.JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
+
+    DatasetService tableService = bqServices.getDatasetService(bqOptions);
+    tableService.deleteTable(tableToRemove);
+    tableService.deleteDataset(tableToRemove.getProjectId(), tableToRemove.getDatasetId());
+  }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+    builder.add(DisplayData.item("query", query));
+  }
+
+  private synchronized JobStatistics dryRunQueryIfNeeded(BigQueryOptions bqOptions)
+      throws InterruptedException, IOException {
+    if (dryRunJobStats.get() == null) {
+      JobStatistics jobStats = bqServices.getJobService(bqOptions).dryRunQuery(
+          executingProject.get(), createBasicQueryConfig());
+      dryRunJobStats.compareAndSet(null, jobStats);
+    }
+    return dryRunJobStats.get();
+  }
+
+  private void executeQuery(
+      String executingProject,
+      String jobId,
+      TableReference destinationTable,
+      JobService jobService) throws IOException, InterruptedException {
+    JobReference jobRef = new JobReference()
+        .setProjectId(executingProject)
+        .setJobId(jobId);
+
+    JobConfigurationQuery queryConfig = createBasicQueryConfig()
+        .setAllowLargeResults(true)
+        .setCreateDisposition("CREATE_IF_NEEDED")
+        .setDestinationTable(destinationTable)
+        .setPriority("BATCH")
+        .setWriteDisposition("WRITE_EMPTY");
+
+    jobService.startQueryJob(jobRef, queryConfig);
+    Job job = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
+    if (BigQueryHelpers.parseStatus(job) != Status.SUCCEEDED) {
+      throw new IOException(String.format(
+          "Query job %s failed, status: %s.", jobId,
+          BigQueryHelpers.statusToPrettyString(job.getStatus())));
+    }
+  }
+
+  private JobConfigurationQuery createBasicQueryConfig() {
+    return new JobConfigurationQuery()
+        .setFlattenResults(flattenResults)
+        .setQuery(query.get())
+        .setUseLegacySql(useLegacySql);
+  }
+
+  private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException {
+    in.defaultReadObject();
+    dryRunJobStats = new AtomicReference<>();
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index 15ca262..c8e6ed8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -444,7 +444,7 @@ class BigQueryServicesImpl implements BigQueryServices {
     @Override
     public void createTable(Table table) throws InterruptedException, IOException {
       LOG.info("Trying to create BigQuery table: {}",
-          BigQueryIO.toTableSpec(table.getTableReference()));
+          BigQueryHelpers.toTableSpec(table.getTableReference()));
       BackOff backoff =
               new ExponentialBackOff.Builder()
                       .setMaxElapsedTimeMillis(RETRY_CREATE_TABLE_DURATION_MILLIS)

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
new file mode 100644
index 0000000..ff50e6d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -0,0 +1,178 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationExtract;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.TableRowJsonCoder;
+import org.apache.beam.sdk.io.AvroSource;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An abstract {@link BoundedSource} to read a table from BigQuery.
+ *
+ * <p>This source uses a BigQuery export job to take a snapshot of the table on GCS, and then
+ * reads in parallel from each produced file. It is implemented by {@link BigQueryTableSource},
+ * and {@link BigQueryQuerySource}, depending on the configuration of the read.
+ * Specifically,
+ * <ul>
+ * <li>{@link BigQueryTableSource} is for reading BigQuery tables</li>
+ * <li>{@link BigQueryQuerySource} is for querying BigQuery tables</li>
+ * </ul>
+ * ...
+ */
+abstract class BigQuerySourceBase extends BoundedSource<TableRow> {
+  private static final Logger LOG = LoggerFactory.getLogger(BigQuerySourceBase.class);
+
+  // The maximum number of retries to poll a BigQuery job.
+  protected static final int JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
+
+  protected final ValueProvider<String> jobIdToken;
+  protected final String extractDestinationDir;
+  protected final BigQueryServices bqServices;
+  protected final ValueProvider<String> executingProject;
+
+  BigQuerySourceBase(
+      ValueProvider<String> jobIdToken,
+      String extractDestinationDir,
+      BigQueryServices bqServices,
+      ValueProvider<String> executingProject) {
+    this.jobIdToken = checkNotNull(jobIdToken, "jobIdToken");
+    this.extractDestinationDir = checkNotNull(extractDestinationDir, "extractDestinationDir");
+    this.bqServices = checkNotNull(bqServices, "bqServices");
+    this.executingProject = checkNotNull(executingProject, "executingProject");
+  }
+
+  @Override
+  public List<BoundedSource<TableRow>> splitIntoBundles(
+      long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+    BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+    TableReference tableToExtract = getTableToExtract(bqOptions);
+    JobService jobService = bqServices.getJobService(bqOptions);
+    String extractJobId = BigQueryIO.getExtractJobId(jobIdToken);
+    List<String> tempFiles = executeExtract(extractJobId, tableToExtract, jobService);
+
+    TableSchema tableSchema = bqServices.getDatasetService(bqOptions)
+        .getTable(tableToExtract).getSchema();
+
+    cleanupTempResource(bqOptions);
+    return createSources(tempFiles, tableSchema);
+  }
+
+  protected abstract TableReference getTableToExtract(BigQueryOptions bqOptions) throws Exception;
+
+  protected abstract void cleanupTempResource(BigQueryOptions bqOptions) throws Exception;
+
+  @Override
+  public void validate() {
+    // Do nothing, validation is done in BigQuery.Read.
+  }
+
+  @Override
+  public Coder<TableRow> getDefaultOutputCoder() {
+    return TableRowJsonCoder.of();
+  }
+
+  private List<String> executeExtract(
+      String jobId, TableReference table, JobService jobService)
+          throws InterruptedException, IOException {
+    JobReference jobRef = new JobReference()
+        .setProjectId(executingProject.get())
+        .setJobId(jobId);
+
+    String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir);
+    JobConfigurationExtract extract = new JobConfigurationExtract()
+        .setSourceTable(table)
+        .setDestinationFormat("AVRO")
+        .setDestinationUris(ImmutableList.of(destinationUri));
+
+    LOG.info("Starting BigQuery extract job: {}", jobId);
+    jobService.startExtractJob(jobRef, extract);
+    Job extractJob =
+        jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
+    if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) {
+      throw new IOException(String.format(
+          "Extract job %s failed, status: %s.",
+          extractJob.getJobReference().getJobId(),
+          BigQueryHelpers.statusToPrettyString(extractJob.getStatus())));
+    }
+
+    List<String> tempFiles = BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob);
+    return ImmutableList.copyOf(tempFiles);
+  }
+
+  private List<BoundedSource<TableRow>> createSources(
+      List<String> files, TableSchema tableSchema) throws IOException, InterruptedException {
+    final String jsonSchema = BigQueryIO.JSON_FACTORY.toString(tableSchema);
+
+    SerializableFunction<GenericRecord, TableRow> function =
+        new SerializableFunction<GenericRecord, TableRow>() {
+          @Override
+          public TableRow apply(GenericRecord input) {
+            return BigQueryAvroUtils.convertGenericRecordToTableRow(
+                input, BigQueryHelpers.fromJsonString(jsonSchema, TableSchema.class));
+          }};
+
+    List<BoundedSource<TableRow>> avroSources = Lists.newArrayList();
+    for (String fileName : files) {
+      avroSources.add(new TransformingSource<>(
+          AvroSource.from(fileName), function, getDefaultOutputCoder()));
+    }
+    return ImmutableList.copyOf(avroSources);
+  }
+
+  protected static class BigQueryReader extends BoundedReader<TableRow> {
+    private final BigQuerySourceBase source;
+    private final BigQueryServices.BigQueryJsonReader reader;
+
+    BigQueryReader(
+        BigQuerySourceBase source, BigQueryServices.BigQueryJsonReader reader) {
+      this.source = source;
+      this.reader = reader;
+    }
+
+    @Override
+    public BoundedSource<TableRow> getCurrentSource() {
+      return source;
+    }
+
+    @Override
+    public boolean start() throws IOException {
+      return reader.start();
+    }
+
+    @Override
+    public boolean advance() throws IOException {
+      return reader.advance();
+    }
+
+    @Override
+    public TableRow getCurrent() throws NoSuchElementException {
+      return reader.getCurrent();
+    }
+
+    @Override
+    public void close() throws IOException {
+      reader.close();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
new file mode 100644
index 0000000..aae0faa
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
@@ -0,0 +1,86 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+/**
+ * A {@link BigQuerySourceBase} for reading BigQuery tables.
+ */
+@VisibleForTesting
+class BigQueryTableSource extends BigQuerySourceBase {
+
+  static BigQueryTableSource create(
+      ValueProvider<String> jobIdToken,
+      ValueProvider<TableReference> table,
+      String extractDestinationDir,
+      BigQueryServices bqServices,
+      ValueProvider<String> executingProject) {
+    return new BigQueryTableSource(
+        jobIdToken, table, extractDestinationDir, bqServices, executingProject);
+  }
+
+  private final ValueProvider<String> jsonTable;
+  private final AtomicReference<Long> tableSizeBytes;
+
+  private BigQueryTableSource(
+      ValueProvider<String> jobIdToken,
+      ValueProvider<TableReference> table,
+      String extractDestinationDir,
+      BigQueryServices bqServices,
+      ValueProvider<String> executingProject) {
+    super(jobIdToken, extractDestinationDir, bqServices, executingProject);
+    this.jsonTable = NestedValueProvider.of(checkNotNull(table, "table"), new TableRefToJson());
+    this.tableSizeBytes = new AtomicReference<>();
+  }
+
+  @Override
+  protected TableReference getTableToExtract(BigQueryOptions bqOptions) throws IOException {
+    checkState(jsonTable.isAccessible());
+    return BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
+  }
+
+  @Override
+  public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
+    BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+    checkState(jsonTable.isAccessible());
+    TableReference tableRef = BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(),
+        TableReference.class);
+    return new BigQueryReader(this, bqServices.getReaderFromTable(bqOptions, tableRef));
+  }
+
+  @Override
+  public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+    if (tableSizeBytes.get() == null) {
+      TableReference table = BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(),
+          TableReference.class);
+
+      Long numBytes = bqServices.getDatasetService(options.as(BigQueryOptions.class))
+          .getTable(table).getNumBytes();
+      tableSizeBytes.compareAndSet(null, numBytes);
+    }
+    return tableSizeBytes.get();
+  }
+
+  @Override
+  protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
+    // Do nothing.
+  }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+    builder.add(DisplayData.item("table", jsonTable));
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
new file mode 100644
index 0000000..612afbe
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
@@ -0,0 +1,66 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+
+/**
+ * A {@link PTransform} that invokes {@link CleanupOperation} after the input {@link PCollection}
+ * has been processed.
+ */
+@VisibleForTesting
+class PassThroughThenCleanup<T> extends PTransform<PCollection<T>, PCollection<T>> {
+
+  private CleanupOperation cleanupOperation;
+
+  PassThroughThenCleanup(CleanupOperation cleanupOperation) {
+    this.cleanupOperation = cleanupOperation;
+  }
+
+  @Override
+  public PCollection<T> expand(PCollection<T> input) {
+    TupleTag<T> mainOutput = new TupleTag<>();
+    TupleTag<Void> cleanupSignal = new TupleTag<>();
+    PCollectionTuple outputs = input.apply(ParDo.of(new IdentityFn<T>())
+        .withOutputTags(mainOutput, TupleTagList.of(cleanupSignal)));
+
+    PCollectionView<Void> cleanupSignalView = outputs.get(cleanupSignal)
+        .setCoder(VoidCoder.of())
+        .apply(View.<Void>asSingleton().withDefaultValue(null));
+
+    input.getPipeline()
+        .apply("Create(CleanupOperation)", Create.of(cleanupOperation))
+        .apply("Cleanup", ParDo.of(
+            new DoFn<CleanupOperation, Void>() {
+              @ProcessElement
+              public void processElement(ProcessContext c)
+                  throws Exception {
+                c.element().cleanup(c.getPipelineOptions());
+              }
+            }).withSideInputs(cleanupSignalView));
+
+    return outputs.get(mainOutput);
+  }
+
+  private static class IdentityFn<T> extends DoFn<T, T> {
+    @ProcessElement
+    public void processElement(ProcessContext c) {
+      c.output(c.element());
+    }
+  }
+
+  abstract static class CleanupOperation implements Serializable {
+    abstract void cleanup(PipelineOptions options) throws Exception;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
new file mode 100644
index 0000000..a86adfb
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
@@ -0,0 +1,118 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.joda.time.Instant;
+
+/**
+ * A {@link BoundedSource} that reads from {@code BoundedSource<T>}
+ * and transforms elements to type {@code V}.
+*/
+@VisibleForTesting
+class TransformingSource<T, V> extends BoundedSource<V> {
+  private final BoundedSource<T> boundedSource;
+  private final SerializableFunction<T, V> function;
+  private final Coder<V> outputCoder;
+
+  TransformingSource(
+      BoundedSource<T> boundedSource,
+      SerializableFunction<T, V> function,
+      Coder<V> outputCoder) {
+    this.boundedSource = checkNotNull(boundedSource, "boundedSource");
+    this.function = checkNotNull(function, "function");
+    this.outputCoder = checkNotNull(outputCoder, "outputCoder");
+  }
+
+  @Override
+  public List<? extends BoundedSource<V>> splitIntoBundles(
+      long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+    return Lists.transform(
+        boundedSource.splitIntoBundles(desiredBundleSizeBytes, options),
+        new Function<BoundedSource<T>, BoundedSource<V>>() {
+          @Override
+          public BoundedSource<V> apply(BoundedSource<T> input) {
+            return new TransformingSource<>(input, function, outputCoder);
+          }
+        });
+  }
+
+  @Override
+  public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+    return boundedSource.getEstimatedSizeBytes(options);
+  }
+
+  @Override
+  public BoundedReader<V> createReader(PipelineOptions options) throws IOException {
+    return new TransformingReader(boundedSource.createReader(options));
+  }
+
+  @Override
+  public void validate() {
+    boundedSource.validate();
+  }
+
+  @Override
+  public Coder<V> getDefaultOutputCoder() {
+    return outputCoder;
+  }
+
+  private class TransformingReader extends BoundedReader<V> {
+    private final BoundedReader<T> boundedReader;
+
+    private TransformingReader(BoundedReader<T> boundedReader) {
+      this.boundedReader = checkNotNull(boundedReader, "boundedReader");
+    }
+
+    @Override
+    public synchronized BoundedSource<V> getCurrentSource() {
+      return new TransformingSource<>(boundedReader.getCurrentSource(), function, outputCoder);
+    }
+
+    @Override
+    public boolean start() throws IOException {
+      return boundedReader.start();
+    }
+
+    @Override
+    public boolean advance() throws IOException {
+      return boundedReader.advance();
+    }
+
+    @Override
+    public V getCurrent() throws NoSuchElementException {
+      T current = boundedReader.getCurrent();
+      return function.apply(current);
+    }
+
+    @Override
+    public void close() throws IOException {
+      boundedReader.close();
+    }
+
+    @Override
+    public synchronized BoundedSource<V> splitAtFraction(double fraction) {
+      BoundedSource<T> split = boundedReader.splitAtFraction(fraction);
+      return split == null ? null : new TransformingSource<>(split, function, outputCoder);
+    }
+
+    @Override
+    public Double getFractionConsumed() {
+      return boundedReader.getFractionConsumed();
+    }
+
+    @Override
+    public Instant getCurrentTimestamp() throws NoSuchElementException {
+      return boundedReader.getCurrentTimestamp();
+    }
+  }
+}


[3/6] beam git commit: Refactor BigQueryIO.Write helper transforms into separate files.

Posted by jk...@apache.org.
Refactor BigQueryIO.Write helper transforms into separate files.


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d6ef0104
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d6ef0104
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d6ef0104

Branch: refs/heads/master
Commit: d6ef0104d5b8e1076cea2a9d08ad14ba0e8e84f1
Parents: 2cc2e81
Author: Reuven Lax <re...@google.com>
Authored: Fri Mar 17 15:30:53 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:15 2017 -0700

----------------------------------------------------------------------
 .../sdk/io/gcp/bigquery/BatchLoadBigQuery.java  |  182 +++
 .../sdk/io/gcp/bigquery/BigQueryHelpers.java    |   64 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    | 1063 +-----------------
 .../io/gcp/bigquery/BigQueryQuerySource.java    |   21 +-
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java |   20 +-
 .../io/gcp/bigquery/BigQueryTableSource.java    |   18 +
 .../io/gcp/bigquery/PassThroughThenCleanup.java |   18 +
 .../beam/sdk/io/gcp/bigquery/ShardedKey.java    |   44 +
 .../sdk/io/gcp/bigquery/ShardedKeyCoder.java    |   87 ++
 .../sdk/io/gcp/bigquery/StreamWithDeDup.java    |   98 ++
 .../sdk/io/gcp/bigquery/StreamingWriteFn.java   |  186 +++
 .../beam/sdk/io/gcp/bigquery/TableRowInfo.java  |   34 +
 .../sdk/io/gcp/bigquery/TableRowInfoCoder.java  |   68 ++
 .../sdk/io/gcp/bigquery/TableRowWriter.java     |   84 ++
 .../gcp/bigquery/TagWithUniqueIdsAndTable.java  |  135 +++
 .../sdk/io/gcp/bigquery/TransformingSource.java |   18 +
 .../beam/sdk/io/gcp/bigquery/WriteBundles.java  |   82 ++
 .../sdk/io/gcp/bigquery/WritePartition.java     |   79 ++
 .../beam/sdk/io/gcp/bigquery/WriteRename.java   |  180 +++
 .../beam/sdk/io/gcp/bigquery/WriteTables.java   |  213 ++++
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |   39 +-
 21 files changed, 1653 insertions(+), 1080 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java
new file mode 100644
index 0000000..75b1cc7
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import java.io.IOException;
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.IOChannelFactory;
+import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+/**
+ * PTransform that uses BigQuery batch-load jobs to write a PCollection to BigQuery.
+ */
+class BatchLoadBigQuery<T> extends PTransform<PCollection<T>, PDone> {
+  BigQueryIO.Write<T> write;
+
+  BatchLoadBigQuery(BigQueryIO.Write<T> write) {
+    this.write = write;
+  }
+
+  @Override
+  public PDone expand(PCollection<T> input) {
+    Pipeline p = input.getPipeline();
+    BigQueryOptions options = p.getOptions().as(BigQueryOptions.class);
+    ValueProvider<TableReference> table = write.getTableWithDefaultProject(options);
+
+    final String stepUuid = BigQueryHelpers.randomUUIDString();
+
+    String tempLocation = options.getTempLocation();
+    String tempFilePrefix;
+    try {
+      IOChannelFactory factory = IOChannelUtils.getFactory(tempLocation);
+      tempFilePrefix = factory.resolve(
+          factory.resolve(tempLocation, "BigQueryWriteTemp"),
+          stepUuid);
+    } catch (IOException e) {
+      throw new RuntimeException(
+          String.format("Failed to resolve BigQuery temp location in %s", tempLocation),
+          e);
+    }
+
+    // Create a singleton job ID token at execution time.
+    PCollection<String> singleton = p.apply("Create", Create.of(tempFilePrefix));
+    PCollectionView<String> jobIdTokenView = p
+        .apply("TriggerIdCreation", Create.of("ignored"))
+        .apply("CreateJobId", MapElements.via(
+            new SimpleFunction<String, String>() {
+              @Override
+              public String apply(String input) {
+                return stepUuid;
+              }
+            }))
+        .apply(View.<String>asSingleton());
+
+    PCollection<T> typedInputInGlobalWindow =
+        input.apply(
+            Window.<T>into(new GlobalWindows())
+                .triggering(DefaultTrigger.of())
+                .discardingFiredPanes());
+    // Avoid applying the formatFunction if it is the identity formatter.
+    PCollection<TableRow> inputInGlobalWindow;
+    if (write.getFormatFunction() == BigQueryIO.IDENTITY_FORMATTER) {
+      inputInGlobalWindow = (PCollection<TableRow>) typedInputInGlobalWindow;
+    } else {
+      inputInGlobalWindow = typedInputInGlobalWindow
+          .apply(MapElements.via(write.getFormatFunction())
+              .withOutputType(new TypeDescriptor<TableRow>() {
+              }));
+    }
+
+    // PCollection of filename, file byte size.
+    PCollection<KV<String, Long>> results = inputInGlobalWindow
+        .apply("WriteBundles",
+            ParDo.of(new WriteBundles(tempFilePrefix)));
+
+    TupleTag<KV<Long, List<String>>> multiPartitionsTag =
+        new TupleTag<KV<Long, List<String>>>("multiPartitionsTag") {};
+    TupleTag<KV<Long, List<String>>> singlePartitionTag =
+        new TupleTag<KV<Long, List<String>>>("singlePartitionTag") {};
+
+    // Turn the list of files and record counts in a PCollectionView that can be used as a
+    // side input.
+    PCollectionView<Iterable<KV<String, Long>>> resultsView = results
+        .apply("ResultsView", View.<KV<String, Long>>asIterable());
+    PCollectionTuple partitions = singleton.apply(ParDo
+        .of(new WritePartition(
+            resultsView,
+            multiPartitionsTag,
+            singlePartitionTag))
+        .withSideInputs(resultsView)
+        .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
+
+    // If WriteBundles produced more than MAX_NUM_FILES files or MAX_SIZE_BYTES bytes, then
+    // the import needs to be split into multiple partitions, and those partitions will be
+    // specified in multiPartitionsTag.
+    PCollection<String> tempTables = partitions.get(multiPartitionsTag)
+        .apply("MultiPartitionsGroupByKey", GroupByKey.<Long, List<String>>create())
+        .apply("MultiPartitionsWriteTables", ParDo.of(new WriteTables(
+            false,
+            write.getBigQueryServices(),
+            jobIdTokenView,
+            tempFilePrefix,
+            NestedValueProvider.of(table, new TableRefToJson()),
+            write.getJsonSchema(),
+            WriteDisposition.WRITE_EMPTY,
+            CreateDisposition.CREATE_IF_NEEDED,
+            write.getTableDescription()))
+            .withSideInputs(jobIdTokenView));
+
+    PCollectionView<Iterable<String>> tempTablesView = tempTables
+        .apply("TempTablesView", View.<String>asIterable());
+    singleton.apply(ParDo
+        .of(new WriteRename(
+            write.getBigQueryServices(),
+            jobIdTokenView,
+            NestedValueProvider.of(table, new TableRefToJson()),
+            write.getWriteDisposition(),
+            write.getCreateDisposition(),
+            tempTablesView,
+            write.getTableDescription()))
+        .withSideInputs(tempTablesView, jobIdTokenView));
+
+    // Write single partition to final table
+    partitions.get(singlePartitionTag)
+        .apply("SinglePartitionGroupByKey", GroupByKey.<Long, List<String>>create())
+        .apply("SinglePartitionWriteTables", ParDo.of(new WriteTables(
+            true,
+            write.getBigQueryServices(),
+            jobIdTokenView,
+            tempFilePrefix,
+            NestedValueProvider.of(table, new TableRefToJson()),
+            write.getJsonSchema(),
+            write.getWriteDisposition(),
+            write.getCreateDisposition(),
+            write.getTableDescription()))
+            .withSideInputs(jobIdTokenView));
+
+    return PDone.in(input.getPipeline());
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 9fba938..37ff124 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -1,5 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.beam.sdk.io.gcp.bigquery;
 
+import static com.google.common.base.Preconditions.checkState;
+
 import com.google.api.services.bigquery.model.Job;
 import com.google.api.services.bigquery.model.JobStatus;
 import com.google.api.services.bigquery.model.TableReference;
@@ -14,16 +34,34 @@ import java.util.UUID;
 import java.util.regex.Matcher;
 import javax.annotation.Nullable;
 
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 
-
 /**
  * A set of helper functions and classes used by {@link BigQueryIO}.
  */
-public class BigQueryHelpers {
+class BigQueryHelpers {
+  private static final String RESOURCE_NOT_FOUND_ERROR =
+      "BigQuery %1$s not found for table \"%2$s\" . Please create the %1$s before pipeline"
+          + " execution. If the %1$s is created by an earlier stage of the pipeline, this"
+          + " validation can be disabled using #withoutValidation.";
+
+  private static final String UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR =
+      "Unable to confirm BigQuery %1$s presence for table \"%2$s\". If the %1$s is created by"
+          + " an earlier stage of the pipeline, this validation can be disabled using"
+          + " #withoutValidation.";
+
+  /**
+   * Status of a BigQuery job or request.
+   */
+  enum Status {
+    SUCCEEDED,
+    FAILED,
+    UNKNOWN,
+  }
+
   @Nullable
   /**
    * Return a displayable string representation for a {@link TableReference}.
@@ -138,6 +176,26 @@ public class BigQueryHelpers {
     return UUID.randomUUID().toString().replaceAll("-", "");
   }
 
+  static void verifyTableNotExistOrEmpty(
+      DatasetService datasetService,
+      TableReference tableRef) {
+    try {
+      if (datasetService.getTable(tableRef) != null) {
+        checkState(
+            datasetService.isTableEmpty(tableRef),
+            "BigQuery table is not empty: %s.",
+            toTableSpec(tableRef));
+      }
+    } catch (IOException | InterruptedException e) {
+      if (e instanceof InterruptedException) {
+        Thread.currentThread().interrupt();
+      }
+      throw new RuntimeException(
+          "unable to confirm BigQuery table emptiness for table "
+              + toTableSpec(tableRef), e);
+    }
+  }
+
   @VisibleForTesting
   static class JsonSchemaToTableSchema
       implements SerializableFunction<String, TableSchema> {

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 3f2f3e8..4917083 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -18,19 +18,13 @@
 package org.apache.beam.sdk.io.gcp.bigquery;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobConfigurationLoad;
 import com.google.api.services.bigquery.model.JobConfigurationQuery;
-import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
 import com.google.api.services.bigquery.model.JobReference;
 import com.google.api.services.bigquery.model.JobStatistics;
-import com.google.api.services.bigquery.model.Table;
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.api.services.bigquery.model.TableSchema;
@@ -39,54 +33,30 @@ import com.google.cloud.hadoop.util.ApiErrorExtractor;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
 import com.google.common.io.CountingOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadLocalRandom;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
 
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.Coder.Context;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StandardCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.TableRowJsonCoder;
-import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.BeamJobUuidToBigQueryJobUuid;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreateJsonTableRefFromUuid;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreatePerBeamJobUuid;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonTableRefToTableRef;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToTableSpec;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
-import org.apache.beam.sdk.metrics.Counter;
-import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.BigQueryOptions;
 import org.apache.beam.sdk.options.GcpOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -94,41 +64,19 @@ import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.SimpleFunction;
-import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.FileIOChannelFactory;
-import org.apache.beam.sdk.util.GcsIOChannelFactory;
-import org.apache.beam.sdk.util.GcsUtil;
 import org.apache.beam.sdk.util.GcsUtil.GcsUtilFactory;
 import org.apache.beam.sdk.util.IOChannelFactory;
 import org.apache.beam.sdk.util.IOChannelUtils;
 import org.apache.beam.sdk.util.MimeTypes;
-import org.apache.beam.sdk.util.PropertyNames;
-import org.apache.beam.sdk.util.Reshuffle;
 import org.apache.beam.sdk.util.Transport;
 import org.apache.beam.sdk.util.gcsfs.GcsPath;
-import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.sdk.values.PDone;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
-import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.sdk.values.ValueInSingleWindow;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -296,7 +244,7 @@ public class BigQueryIO {
    * A formatting function that maps a TableRow to itself. This allows sending a
    * {@code PCollection<TableRow>} directly to BigQueryIO.Write.
    */
-  private static final SerializableFunction<TableRow, TableRow> IDENTITY_FORMATTER =
+   static final SerializableFunction<TableRow, TableRow> IDENTITY_FORMATTER =
       new SerializableFunction<TableRow, TableRow>() {
     @Override
     public TableRow apply(TableRow input) {
@@ -622,7 +570,7 @@ public class BigQueryIO {
      *
      * <p>If the table's project is not specified, use the executing project.
      */
-    @Nullable private ValueProvider<TableReference> getTableWithDefaultProject(
+    @Nullable ValueProvider<TableReference> getTableWithDefaultProject(
         BigQueryOptions bqOptions) {
       ValueProvider<TableReference> table = getTableProvider();
       if (table == null) {
@@ -753,11 +701,11 @@ public class BigQueryIO {
     static final long MAX_SIZE_BYTES = 11 * (1L << 40);
 
     // The maximum number of retry jobs.
-    private static final int MAX_RETRY_JOBS = 3;
+    static final int MAX_RETRY_JOBS = 3;
 
     // The maximum number of retries to poll the status of a job.
     // It sets to {@code Integer.MAX_VALUE} to block until the BigQuery job finishes.
-    private static final int LOAD_JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
+    static final int LOAD_JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
 
     @Nullable abstract ValueProvider<String> getJsonTableRef();
     @Nullable abstract SerializableFunction<ValueInSingleWindow<T>, TableReference>
@@ -974,26 +922,6 @@ public class BigQueryIO {
       return toBuilder().setBigQueryServices(testServices).build();
     }
 
-    private static void verifyTableNotExistOrEmpty(
-        DatasetService datasetService,
-        TableReference tableRef) {
-      try {
-        if (datasetService.getTable(tableRef) != null) {
-          checkState(
-              datasetService.isTableEmpty(tableRef),
-              "BigQuery table is not empty: %s.",
-              BigQueryHelpers.toTableSpec(tableRef));
-        }
-      } catch (IOException | InterruptedException e) {
-        if (e instanceof InterruptedException) {
-          Thread.currentThread().interrupt();
-        }
-        throw new RuntimeException(
-            "unable to confirm BigQuery table emptiness for table "
-                + BigQueryHelpers.toTableSpec(tableRef), e);
-      }
-    }
-
     @Override
     public void validate(PCollection<T> input) {
       BigQueryOptions options = input.getPipeline().getOptions().as(BigQueryOptions.class);
@@ -1030,7 +958,7 @@ public class BigQueryIO {
           verifyTablePresence(datasetService, table);
         }
         if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
-          verifyTableNotExistOrEmpty(datasetService, table);
+          BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table);
         }
       }
 
@@ -1074,176 +1002,12 @@ public class BigQueryIO {
 
     @Override
     public PDone expand(PCollection<T> input) {
-      Pipeline p = input.getPipeline();
-      BigQueryOptions options = p.getOptions().as(BigQueryOptions.class);
-
       // When writing an Unbounded PCollection, or when a tablespec function is defined, we use
       // StreamWithDeDup and BigQuery's streaming import API.
       if (input.isBounded() == IsBounded.UNBOUNDED || getTableRefFunction() != null) {
         return input.apply(new StreamWithDeDup<T>(this));
-      }
-
-      ValueProvider<TableReference> table = getTableWithDefaultProject(options);
-
-      final String stepUuid = BigQueryHelpers.randomUUIDString();
-
-      String tempLocation = options.getTempLocation();
-      String tempFilePrefix;
-      try {
-        IOChannelFactory factory = IOChannelUtils.getFactory(tempLocation);
-        tempFilePrefix = factory.resolve(
-                factory.resolve(tempLocation, "BigQueryWriteTemp"),
-                stepUuid);
-      } catch (IOException e) {
-        throw new RuntimeException(
-            String.format("Failed to resolve BigQuery temp location in %s", tempLocation),
-            e);
-      }
-
-      // Create a singleton job ID token at execution time.
-      PCollection<String> singleton = p.apply("Create", Create.of(tempFilePrefix));
-      PCollectionView<String> jobIdTokenView = p
-          .apply("TriggerIdCreation", Create.of("ignored"))
-          .apply("CreateJobId", MapElements.via(
-              new SimpleFunction<String, String>() {
-                @Override
-                public String apply(String input) {
-                  return stepUuid;
-                }
-              }))
-          .apply(View.<String>asSingleton());
-
-      PCollection<T> typedInputInGlobalWindow =
-          input.apply(
-              Window.<T>into(new GlobalWindows())
-                  .triggering(DefaultTrigger.of())
-                  .discardingFiredPanes());
-      // Avoid applying the formatFunction if it is the identity formatter.
-      PCollection<TableRow> inputInGlobalWindow;
-      if (getFormatFunction() == IDENTITY_FORMATTER) {
-        inputInGlobalWindow = (PCollection<TableRow>) typedInputInGlobalWindow;
       } else {
-        inputInGlobalWindow = typedInputInGlobalWindow
-            .apply(MapElements.via(getFormatFunction())
-                .withOutputType(new TypeDescriptor<TableRow>() {
-                }));
-      }
-
-      // PCollection of filename, file byte size.
-      PCollection<KV<String, Long>> results = inputInGlobalWindow
-          .apply("WriteBundles",
-              ParDo.of(new WriteBundles(tempFilePrefix)));
-
-      TupleTag<KV<Long, List<String>>> multiPartitionsTag =
-          new TupleTag<KV<Long, List<String>>>("multiPartitionsTag") {};
-      TupleTag<KV<Long, List<String>>> singlePartitionTag =
-          new TupleTag<KV<Long, List<String>>>("singlePartitionTag") {};
-
-      // Turn the list of files and record counts in a PCollectionView that can be used as a
-      // side input.
-      PCollectionView<Iterable<KV<String, Long>>> resultsView = results
-          .apply("ResultsView", View.<KV<String, Long>>asIterable());
-      PCollectionTuple partitions = singleton.apply(ParDo
-          .of(new WritePartition(
-              resultsView,
-              multiPartitionsTag,
-              singlePartitionTag))
-          .withSideInputs(resultsView)
-          .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
-
-      // If WriteBundles produced more than MAX_NUM_FILES files or MAX_SIZE_BYTES bytes, then
-      // the import needs to be split into multiple partitions, and those partitions will be
-      // specified in multiPartitionsTag.
-      PCollection<String> tempTables = partitions.get(multiPartitionsTag)
-          .apply("MultiPartitionsGroupByKey", GroupByKey.<Long, List<String>>create())
-          .apply("MultiPartitionsWriteTables", ParDo.of(new WriteTables(
-              false,
-              getBigQueryServices(),
-              jobIdTokenView,
-              tempFilePrefix,
-              NestedValueProvider.of(table, new TableRefToJson()),
-              getJsonSchema(),
-              WriteDisposition.WRITE_EMPTY,
-              CreateDisposition.CREATE_IF_NEEDED,
-              getTableDescription()))
-          .withSideInputs(jobIdTokenView));
-
-      PCollectionView<Iterable<String>> tempTablesView = tempTables
-          .apply("TempTablesView", View.<String>asIterable());
-      singleton.apply(ParDo
-          .of(new WriteRename(
-              getBigQueryServices(),
-              jobIdTokenView,
-              NestedValueProvider.of(table, new TableRefToJson()),
-              getWriteDisposition(),
-              getCreateDisposition(),
-              tempTablesView,
-              getTableDescription()))
-          .withSideInputs(tempTablesView, jobIdTokenView));
-
-      // Write single partition to final table
-      partitions.get(singlePartitionTag)
-          .apply("SinglePartitionGroupByKey", GroupByKey.<Long, List<String>>create())
-          .apply("SinglePartitionWriteTables", ParDo.of(new WriteTables(
-              true,
-              getBigQueryServices(),
-              jobIdTokenView,
-              tempFilePrefix,
-              NestedValueProvider.of(table, new TableRefToJson()),
-              getJsonSchema(),
-              getWriteDisposition(),
-              getCreateDisposition(),
-              getTableDescription()))
-          .withSideInputs(jobIdTokenView));
-
-      return PDone.in(input.getPipeline());
-    }
-
-    private static class WriteBundles extends DoFn<TableRow, KV<String, Long>> {
-      private transient TableRowWriter writer = null;
-      private final String tempFilePrefix;
-
-      WriteBundles(String tempFilePrefix) {
-        this.tempFilePrefix = tempFilePrefix;
-      }
-
-      @ProcessElement
-      public void processElement(ProcessContext c) throws Exception {
-        if (writer == null) {
-          writer = new TableRowWriter(tempFilePrefix);
-          writer.open(UUID.randomUUID().toString());
-          LOG.debug("Done opening writer {}", writer);
-        }
-        try {
-          writer.write(c.element());
-        } catch (Exception e) {
-          // Discard write result and close the write.
-          try {
-            writer.close();
-            // The writer does not need to be reset, as this DoFn cannot be reused.
-          } catch (Exception closeException) {
-            // Do not mask the exception that caused the write to fail.
-            e.addSuppressed(closeException);
-          }
-          throw e;
-        }
-      }
-
-      @FinishBundle
-      public void finishBundle(Context c) throws Exception {
-        if (writer != null) {
-          c.output(writer.close());
-          writer = null;
-        }
-      }
-
-      @Override
-      public void populateDisplayData(DisplayData.Builder builder) {
-        super.populateDisplayData(builder);
-
-        builder
-            .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
-                .withLabel("Temporary File Prefix"));
+        return input.apply(new BatchLoadBigQuery<T>(this));
       }
     }
 
@@ -1289,7 +1053,7 @@ public class BigQueryIO {
      *
      * <p>If the table's project is not specified, use the executing project.
      */
-    @Nullable private ValueProvider<TableReference> getTableWithDefaultProject(
+    @Nullable ValueProvider<TableReference> getTableWithDefaultProject(
         BigQueryOptions bqOptions) {
       ValueProvider<TableReference> table = getTable();
       if (table == null) {
@@ -1319,391 +1083,6 @@ public class BigQueryIO {
     }
 
 
-    static class TableRowWriter {
-      private static final Coder<TableRow> CODER = TableRowJsonCoder.of();
-      private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
-      private final String tempFilePrefix;
-      private String id;
-      private String fileName;
-      private WritableByteChannel channel;
-      protected String mimeType = MimeTypes.TEXT;
-      private CountingOutputStream out;
-
-      TableRowWriter(String basename) {
-        this.tempFilePrefix = basename;
-      }
-
-      public final void open(String uId) throws Exception {
-        id = uId;
-        fileName = tempFilePrefix + id;
-        LOG.debug("Opening {}.", fileName);
-        channel = IOChannelUtils.create(fileName, mimeType);
-        try {
-          out = new CountingOutputStream(Channels.newOutputStream(channel));
-          LOG.debug("Writing header to {}.", fileName);
-        } catch (Exception e) {
-          try {
-            LOG.error("Writing header to {} failed, closing channel.", fileName);
-            channel.close();
-          } catch (IOException closeException) {
-            LOG.error("Closing channel for {} failed", fileName);
-          }
-          throw e;
-        }
-        LOG.debug("Starting write of bundle {} to {}.", this.id, fileName);
-      }
-
-      public void write(TableRow value) throws Exception {
-        CODER.encode(value, out, Context.OUTER);
-        out.write(NEWLINE);
-      }
-
-      public final KV<String, Long> close() throws IOException {
-        channel.close();
-        return KV.of(fileName, out.getCount());
-      }
-    }
-
-    /**
-     * Partitions temporary files based on number of files and file sizes.
-     */
-    static class WritePartition extends DoFn<String, KV<Long, List<String>>> {
-      private final PCollectionView<Iterable<KV<String, Long>>> resultsView;
-      private TupleTag<KV<Long, List<String>>> multiPartitionsTag;
-      private TupleTag<KV<Long, List<String>>> singlePartitionTag;
-
-      public WritePartition(
-          PCollectionView<Iterable<KV<String, Long>>> resultsView,
-          TupleTag<KV<Long, List<String>>> multiPartitionsTag,
-          TupleTag<KV<Long, List<String>>> singlePartitionTag) {
-        this.resultsView = resultsView;
-        this.multiPartitionsTag = multiPartitionsTag;
-        this.singlePartitionTag = singlePartitionTag;
-      }
-
-      @ProcessElement
-      public void processElement(ProcessContext c) throws Exception {
-        List<KV<String, Long>> results = Lists.newArrayList(c.sideInput(resultsView));
-        if (results.isEmpty()) {
-          TableRowWriter writer = new TableRowWriter(c.element());
-          writer.open(UUID.randomUUID().toString());
-          results.add(writer.close());
-        }
-
-        long partitionId = 0;
-        int currNumFiles = 0;
-        long currSizeBytes = 0;
-        List<String> currResults = Lists.newArrayList();
-        for (int i = 0; i < results.size(); ++i) {
-          KV<String, Long> fileResult = results.get(i);
-          if (currNumFiles + 1 > Write.MAX_NUM_FILES
-              || currSizeBytes + fileResult.getValue() > Write.MAX_SIZE_BYTES) {
-            c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
-            currResults = Lists.newArrayList();
-            currNumFiles = 0;
-            currSizeBytes = 0;
-          }
-          ++currNumFiles;
-          currSizeBytes += fileResult.getValue();
-          currResults.add(fileResult.getKey());
-        }
-        if (partitionId == 0) {
-          c.sideOutput(singlePartitionTag, KV.of(++partitionId, currResults));
-        } else {
-          c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
-        }
-      }
-    }
-
-    /**
-     * Writes partitions to BigQuery tables.
-     */
-    static class WriteTables extends DoFn<KV<Long, Iterable<List<String>>>, String> {
-      private final boolean singlePartition;
-      private final BigQueryServices bqServices;
-      private final PCollectionView<String> jobIdToken;
-      private final String tempFilePrefix;
-      private final ValueProvider<String> jsonTableRef;
-      private final ValueProvider<String> jsonSchema;
-      private final WriteDisposition writeDisposition;
-      private final CreateDisposition createDisposition;
-      @Nullable private final String tableDescription;
-
-      public WriteTables(
-          boolean singlePartition,
-          BigQueryServices bqServices,
-          PCollectionView<String> jobIdToken,
-          String tempFilePrefix,
-          ValueProvider<String> jsonTableRef,
-          ValueProvider<String> jsonSchema,
-          WriteDisposition writeDisposition,
-          CreateDisposition createDisposition,
-          @Nullable String tableDescription) {
-        this.singlePartition = singlePartition;
-        this.bqServices = bqServices;
-        this.jobIdToken = jobIdToken;
-        this.tempFilePrefix = tempFilePrefix;
-        this.jsonTableRef = jsonTableRef;
-        this.jsonSchema = jsonSchema;
-        this.writeDisposition = writeDisposition;
-        this.createDisposition = createDisposition;
-        this.tableDescription = tableDescription;
-      }
-
-      @ProcessElement
-      public void processElement(ProcessContext c) throws Exception {
-        List<String> partition = Lists.newArrayList(c.element().getValue()).get(0);
-        String jobIdPrefix = String.format(
-            c.sideInput(jobIdToken) + "_%05d", c.element().getKey());
-        TableReference ref = BigQueryHelpers.fromJsonString(jsonTableRef.get(),
-            TableReference.class);
-        if (!singlePartition) {
-          ref.setTableId(jobIdPrefix);
-        }
-
-        load(
-            bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
-            bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
-            jobIdPrefix,
-            ref,
-            BigQueryHelpers.fromJsonString(
-                jsonSchema == null ? null : jsonSchema.get(), TableSchema.class),
-            partition,
-            writeDisposition,
-            createDisposition,
-            tableDescription);
-        c.output(BigQueryHelpers.toJsonString(ref));
-
-        removeTemporaryFiles(c.getPipelineOptions(), tempFilePrefix, partition);
-      }
-
-      private void load(
-          JobService jobService,
-          DatasetService datasetService,
-          String jobIdPrefix,
-          TableReference ref,
-          @Nullable TableSchema schema,
-          List<String> gcsUris,
-          WriteDisposition writeDisposition,
-          CreateDisposition createDisposition,
-          @Nullable String tableDescription) throws InterruptedException, IOException {
-        JobConfigurationLoad loadConfig = new JobConfigurationLoad()
-            .setDestinationTable(ref)
-            .setSchema(schema)
-            .setSourceUris(gcsUris)
-            .setWriteDisposition(writeDisposition.name())
-            .setCreateDisposition(createDisposition.name())
-            .setSourceFormat("NEWLINE_DELIMITED_JSON");
-
-        String projectId = ref.getProjectId();
-        Job lastFailedLoadJob = null;
-        for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
-          String jobId = jobIdPrefix + "-" + i;
-          JobReference jobRef = new JobReference()
-              .setProjectId(projectId)
-              .setJobId(jobId);
-          jobService.startLoadJob(jobRef, loadConfig);
-          Job loadJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
-          Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
-          switch (jobStatus) {
-            case SUCCEEDED:
-              if (tableDescription != null) {
-                datasetService.patchTableDescription(ref, tableDescription);
-              }
-              return;
-            case UNKNOWN:
-              throw new RuntimeException(String.format(
-                  "UNKNOWN status of load job [%s]: %s.", jobId,
-                  BigQueryHelpers.jobToPrettyString(loadJob)));
-            case FAILED:
-              lastFailedLoadJob = loadJob;
-              continue;
-            default:
-              throw new IllegalStateException(String.format(
-                  "Unexpected status [%s] of load job: %s.",
-                  jobStatus, BigQueryHelpers.jobToPrettyString(loadJob)));
-          }
-        }
-        throw new RuntimeException(String.format(
-            "Failed to create load job with id prefix %s, "
-                + "reached max retries: %d, last failed load job: %s.",
-            jobIdPrefix,
-            Write.MAX_RETRY_JOBS,
-            BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
-      }
-
-      static void removeTemporaryFiles(
-          PipelineOptions options,
-          String tempFilePrefix,
-          Collection<String> files)
-          throws IOException {
-        IOChannelFactory factory = IOChannelUtils.getFactory(tempFilePrefix);
-        if (factory instanceof GcsIOChannelFactory) {
-          GcsUtil gcsUtil = new GcsUtil.GcsUtilFactory().create(options);
-          gcsUtil.remove(files);
-        } else if (factory instanceof FileIOChannelFactory) {
-          for (String filename : files) {
-            LOG.debug("Removing file {}", filename);
-            boolean exists = Files.deleteIfExists(Paths.get(filename));
-            if (!exists) {
-              LOG.debug("{} does not exist.", filename);
-            }
-          }
-        } else {
-          throw new IOException("Unrecognized file system.");
-        }
-      }
-
-      @Override
-      public void populateDisplayData(DisplayData.Builder builder) {
-        super.populateDisplayData(builder);
-
-        builder
-            .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
-                .withLabel("Temporary File Prefix"))
-            .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
-                .withLabel("Table Reference"))
-            .addIfNotNull(DisplayData.item("jsonSchema", jsonSchema)
-                .withLabel("Table Schema"))
-            .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
-                .withLabel("Table Description"));
-      }
-    }
-
-    /**
-     * Copies temporary tables to destination table.
-     */
-    static class WriteRename extends DoFn<String, Void> {
-      private final BigQueryServices bqServices;
-      private final PCollectionView<String> jobIdToken;
-      private final ValueProvider<String> jsonTableRef;
-      private final WriteDisposition writeDisposition;
-      private final CreateDisposition createDisposition;
-      private final PCollectionView<Iterable<String>> tempTablesView;
-      @Nullable private final String tableDescription;
-
-      public WriteRename(
-          BigQueryServices bqServices,
-          PCollectionView<String> jobIdToken,
-          ValueProvider<String> jsonTableRef,
-          WriteDisposition writeDisposition,
-          CreateDisposition createDisposition,
-          PCollectionView<Iterable<String>> tempTablesView,
-          @Nullable String tableDescription) {
-        this.bqServices = bqServices;
-        this.jobIdToken = jobIdToken;
-        this.jsonTableRef = jsonTableRef;
-        this.writeDisposition = writeDisposition;
-        this.createDisposition = createDisposition;
-        this.tempTablesView = tempTablesView;
-        this.tableDescription = tableDescription;
-      }
-
-      @ProcessElement
-      public void processElement(ProcessContext c) throws Exception {
-        List<String> tempTablesJson = Lists.newArrayList(c.sideInput(tempTablesView));
-
-        // Do not copy if no temp tables are provided
-        if (tempTablesJson.size() == 0) {
-          return;
-        }
-
-        List<TableReference> tempTables = Lists.newArrayList();
-        for (String table : tempTablesJson) {
-          tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
-        }
-        copy(
-            bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
-            bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
-            c.sideInput(jobIdToken),
-            BigQueryHelpers.fromJsonString(jsonTableRef.get(), TableReference.class),
-            tempTables,
-            writeDisposition,
-            createDisposition,
-            tableDescription);
-
-        DatasetService tableService =
-            bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class));
-        removeTemporaryTables(tableService, tempTables);
-      }
-
-      private void copy(
-          JobService jobService,
-          DatasetService datasetService,
-          String jobIdPrefix,
-          TableReference ref,
-          List<TableReference> tempTables,
-          WriteDisposition writeDisposition,
-          CreateDisposition createDisposition,
-          @Nullable String tableDescription) throws InterruptedException, IOException {
-        JobConfigurationTableCopy copyConfig = new JobConfigurationTableCopy()
-            .setSourceTables(tempTables)
-            .setDestinationTable(ref)
-            .setWriteDisposition(writeDisposition.name())
-            .setCreateDisposition(createDisposition.name());
-
-        String projectId = ref.getProjectId();
-        Job lastFailedCopyJob = null;
-        for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
-          String jobId = jobIdPrefix + "-" + i;
-          JobReference jobRef = new JobReference()
-              .setProjectId(projectId)
-              .setJobId(jobId);
-          jobService.startCopyJob(jobRef, copyConfig);
-          Job copyJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
-          Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
-          switch (jobStatus) {
-            case SUCCEEDED:
-              if (tableDescription != null) {
-                datasetService.patchTableDescription(ref, tableDescription);
-              }
-              return;
-            case UNKNOWN:
-              throw new RuntimeException(String.format(
-                  "UNKNOWN status of copy job [%s]: %s.", jobId,
-                  BigQueryHelpers.jobToPrettyString(copyJob)));
-            case FAILED:
-              lastFailedCopyJob = copyJob;
-              continue;
-            default:
-              throw new IllegalStateException(String.format(
-                  "Unexpected status [%s] of load job: %s.",
-                  jobStatus, BigQueryHelpers.jobToPrettyString(copyJob)));
-          }
-        }
-        throw new RuntimeException(String.format(
-            "Failed to create copy job with id prefix %s, "
-                + "reached max retries: %d, last failed copy job: %s.",
-            jobIdPrefix,
-            Write.MAX_RETRY_JOBS,
-            BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
-      }
-
-      static void removeTemporaryTables(DatasetService tableService,
-          List<TableReference> tempTables) {
-        for (TableReference tableRef : tempTables) {
-          try {
-            LOG.debug("Deleting table {}", BigQueryHelpers.toJsonString(tableRef));
-            tableService.deleteTable(tableRef);
-          } catch (Exception e) {
-            LOG.warn("Failed to delete the table {}", BigQueryHelpers.toJsonString(tableRef), e);
-          }
-        }
-      }
-
-      @Override
-      public void populateDisplayData(DisplayData.Builder builder) {
-        super.populateDisplayData(builder);
-
-        builder
-            .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
-                .withLabel("Table Reference"))
-            .add(DisplayData.item("writeDisposition", writeDisposition.toString())
-                .withLabel("Write Disposition"))
-            .add(DisplayData.item("createDisposition", createDisposition.toString())
-                .withLabel("Create Disposition"));
-      }
-    }
   }
 
   private static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
@@ -1753,434 +1132,6 @@ public class BigQueryIO {
   static void clearCreatedTables() {
     StreamingWriteFn.clearCreatedTables();
   }
-  /////////////////////////////////////////////////////////////////////////////
-
-  /**
-   * Implementation of DoFn to perform streaming BigQuery write.
-   */
-  @VisibleForTesting
-  static class StreamingWriteFn
-      extends DoFn<KV<ShardedKey<String>, TableRowInfo>, Void> {
-    /** TableSchema in JSON. Use String to make the class Serializable. */
-    @Nullable private final ValueProvider<String> jsonTableSchema;
-
-    @Nullable private final String tableDescription;
-
-    private final BigQueryServices bqServices;
-
-    /** JsonTableRows to accumulate BigQuery rows in order to batch writes. */
-    private transient Map<String, List<TableRow>> tableRows;
-
-    private final Write.CreateDisposition createDisposition;
-
-    /** The list of unique ids for each BigQuery table row. */
-    private transient Map<String, List<String>> uniqueIdsForTableRows;
-
-    /** The list of tables created so far, so we don't try the creation
-        each time. */
-    private static Set<String> createdTables =
-        Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
-
-    /** Tracks bytes written, exposed as "ByteCount" Metrics Counter. */
-    private Counter byteCounter = Metrics.counter(StreamingWriteFn.class, "ByteCount");
-
-    /** Constructor. */
-    StreamingWriteFn(@Nullable ValueProvider<TableSchema> schema,
-                     Write.CreateDisposition createDisposition,
-                     @Nullable String tableDescription, BigQueryServices bqServices) {
-      this.jsonTableSchema = schema == null ? null :
-          NestedValueProvider.of(schema, new TableSchemaToJsonSchema());
-      this.createDisposition = createDisposition;
-      this.bqServices = checkNotNull(bqServices, "bqServices");
-      this.tableDescription = tableDescription;
-    }
-
-    /**
-     * Clear the cached map of created tables. Used for testing.
-     */
-    private static void clearCreatedTables() {
-      synchronized (createdTables) {
-        createdTables.clear();
-      }
-    }
-
-    /** Prepares a target BigQuery table. */
-    @StartBundle
-    public void startBundle(Context context) {
-      tableRows = new HashMap<>();
-      uniqueIdsForTableRows = new HashMap<>();
-    }
-
-    /** Accumulates the input into JsonTableRows and uniqueIdsForTableRows. */
-    @ProcessElement
-    public void processElement(ProcessContext context) {
-      String tableSpec = context.element().getKey().getKey();
-      List<TableRow> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
-      List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows,
-          tableSpec);
-
-      rows.add(context.element().getValue().tableRow);
-      uniqueIds.add(context.element().getValue().uniqueId);
-    }
-
-    /** Writes the accumulated rows into BigQuery with streaming API. */
-    @FinishBundle
-    public void finishBundle(Context context) throws Exception {
-      BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class);
-
-      for (Map.Entry<String, List<TableRow>> entry : tableRows.entrySet()) {
-        TableReference tableReference = getOrCreateTable(options, entry.getKey());
-        flushRows(tableReference, entry.getValue(),
-            uniqueIdsForTableRows.get(entry.getKey()), options);
-      }
-      tableRows.clear();
-      uniqueIdsForTableRows.clear();
-    }
-
-    @Override
-    public void populateDisplayData(DisplayData.Builder builder) {
-      super.populateDisplayData(builder);
-
-      builder
-          .addIfNotNull(DisplayData.item("schema", jsonTableSchema)
-            .withLabel("Table Schema"))
-          .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
-            .withLabel("Table Description"));
-    }
-
-    public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
-        throws InterruptedException, IOException {
-      TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec);
-      if (createDisposition != createDisposition.CREATE_NEVER
-          && !createdTables.contains(tableSpec)) {
-        synchronized (createdTables) {
-          // Another thread may have succeeded in creating the table in the meanwhile, so
-          // check again. This check isn't needed for correctness, but we add it to prevent
-          // every thread from attempting a create and overwhelming our BigQuery quota.
-          DatasetService datasetService = bqServices.getDatasetService(options);
-          if (!createdTables.contains(tableSpec)) {
-            if (datasetService.getTable(tableReference) == null) {
-              TableSchema tableSchema = JSON_FACTORY.fromString(
-                  jsonTableSchema.get(), TableSchema.class);
-              datasetService.createTable(
-                  new Table()
-                      .setTableReference(tableReference)
-                      .setSchema(tableSchema)
-                      .setDescription(tableDescription));
-            }
-            createdTables.add(tableSpec);
-          }
-        }
-      }
-      return tableReference;
-    }
-
-    /**
-     * Writes the accumulated rows into BigQuery with streaming API.
-     */
-    private void flushRows(TableReference tableReference,
-        List<TableRow> tableRows, List<String> uniqueIds, BigQueryOptions options)
-            throws InterruptedException {
-      if (!tableRows.isEmpty()) {
-        try {
-          long totalBytes = bqServices.getDatasetService(options).insertAll(
-              tableReference, tableRows, uniqueIds);
-          byteCounter.inc(totalBytes);
-        } catch (IOException e) {
-          throw new RuntimeException(e);
-        }
-      }
-    }
-  }
-
-  private static class ShardedKey<K> {
-    private final K key;
-    private final int shardNumber;
-
-    public static <K> ShardedKey<K> of(K key, int shardNumber) {
-      return new ShardedKey<>(key, shardNumber);
-    }
-
-    private ShardedKey(K key, int shardNumber) {
-      this.key = key;
-      this.shardNumber = shardNumber;
-    }
-
-    public K getKey() {
-      return key;
-    }
-
-    public int getShardNumber() {
-      return shardNumber;
-    }
-  }
-
-  /**
-   * A {@link Coder} for {@link ShardedKey}, using a wrapped key {@link Coder}.
-   */
-  @VisibleForTesting
-  static class ShardedKeyCoder<KeyT>
-      extends StandardCoder<ShardedKey<KeyT>> {
-    public static <KeyT> ShardedKeyCoder<KeyT> of(Coder<KeyT> keyCoder) {
-      return new ShardedKeyCoder<>(keyCoder);
-    }
-
-    @JsonCreator
-    public static <KeyT> ShardedKeyCoder<KeyT> of(
-         @JsonProperty(PropertyNames.COMPONENT_ENCODINGS)
-        List<Coder<KeyT>> components) {
-      checkArgument(components.size() == 1, "Expecting 1 component, got %s", components.size());
-      return of(components.get(0));
-    }
-
-    protected ShardedKeyCoder(Coder<KeyT> keyCoder) {
-      this.keyCoder = keyCoder;
-      this.shardNumberCoder = VarIntCoder.of();
-    }
-
-    @Override
-    public List<? extends Coder<?>> getCoderArguments() {
-      return Arrays.asList(keyCoder);
-    }
-
-    @Override
-    public void encode(ShardedKey<KeyT> key, OutputStream outStream, Context context)
-        throws IOException {
-      keyCoder.encode(key.getKey(), outStream, context.nested());
-      shardNumberCoder.encode(key.getShardNumber(), outStream, context);
-    }
-
-    @Override
-    public ShardedKey<KeyT> decode(InputStream inStream, Context context)
-        throws IOException {
-      return new ShardedKey<>(
-          keyCoder.decode(inStream, context.nested()),
-          shardNumberCoder.decode(inStream, context));
-    }
-
-    @Override
-    public void verifyDeterministic() throws NonDeterministicException {
-      keyCoder.verifyDeterministic();
-    }
-
-    Coder<KeyT> keyCoder;
-    VarIntCoder shardNumberCoder;
-  }
-
-  @VisibleForTesting
-  static class TableRowInfoCoder extends AtomicCoder<TableRowInfo> {
-    private static final TableRowInfoCoder INSTANCE = new TableRowInfoCoder();
-
-    @JsonCreator
-    public static TableRowInfoCoder of() {
-      return INSTANCE;
-    }
-
-    @Override
-    public void encode(TableRowInfo value, OutputStream outStream, Context context)
-      throws IOException {
-      if (value == null) {
-        throw new CoderException("cannot encode a null value");
-      }
-      tableRowCoder.encode(value.tableRow, outStream, context.nested());
-      idCoder.encode(value.uniqueId, outStream, context);
-    }
-
-    @Override
-    public TableRowInfo decode(InputStream inStream, Context context)
-      throws IOException {
-      return new TableRowInfo(
-          tableRowCoder.decode(inStream, context.nested()),
-          idCoder.decode(inStream, context));
-    }
-
-    @Override
-    public void verifyDeterministic() throws NonDeterministicException {
-      throw new NonDeterministicException(this, "TableRows are not deterministic.");
-    }
-
-    TableRowJsonCoder tableRowCoder = TableRowJsonCoder.of();
-    StringUtf8Coder idCoder = StringUtf8Coder.of();
-  }
-
-  private static class TableRowInfo {
-    TableRowInfo(TableRow tableRow, String uniqueId) {
-      this.tableRow = tableRow;
-      this.uniqueId = uniqueId;
-    }
-
-    final TableRow tableRow;
-    final String uniqueId;
-  }
-
-  /////////////////////////////////////////////////////////////////////////////
-
-  /**
-   * Fn that tags each table row with a unique id and destination table.
-   * To avoid calling UUID.randomUUID() for each element, which can be costly,
-   * a randomUUID is generated only once per bucket of data. The actual unique
-   * id is created by concatenating this randomUUID with a sequential number.
-   */
-  @VisibleForTesting
-  static class TagWithUniqueIdsAndTable<T>
-      extends DoFn<T, KV<ShardedKey<String>, TableRowInfo>> {
-    /** TableSpec to write to. */
-    private final ValueProvider<String> tableSpec;
-
-    /** User function mapping windowed values to {@link TableReference} in JSON. */
-    private final SerializableFunction<ValueInSingleWindow<T>, TableReference> tableRefFunction;
-
-    /** User function mapping user type to a TableRow. */
-    private final SerializableFunction<T, TableRow> formatFunction;
-
-    private transient String randomUUID;
-    private transient long sequenceNo = 0L;
-
-    TagWithUniqueIdsAndTable(BigQueryOptions options,
-        ValueProvider<TableReference> table,
-        SerializableFunction<ValueInSingleWindow<T>, TableReference> tableRefFunction,
-        SerializableFunction<T, TableRow> formatFunction) {
-      checkArgument(table == null ^ tableRefFunction == null,
-          "Exactly one of table or tableRefFunction should be set");
-      if (table != null) {
-        if (table.isAccessible() && Strings.isNullOrEmpty(table.get().getProjectId())) {
-          TableReference tableRef = table.get()
-              .setProjectId(options.as(BigQueryOptions.class).getProject());
-          table = NestedValueProvider.of(
-              StaticValueProvider.of(BigQueryHelpers.toJsonString(tableRef)),
-              new JsonTableRefToTableRef());
-        }
-        this.tableSpec = NestedValueProvider.of(table, new TableRefToTableSpec());
-      } else {
-        tableSpec = null;
-      }
-      this.tableRefFunction = tableRefFunction;
-      this.formatFunction = formatFunction;
-    }
-
-
-    @StartBundle
-    public void startBundle(Context context) {
-      randomUUID = UUID.randomUUID().toString();
-    }
-
-    /** Tag the input with a unique id. */
-    @ProcessElement
-    public void processElement(ProcessContext context, BoundedWindow window) throws IOException {
-      String uniqueId = randomUUID + sequenceNo++;
-      ThreadLocalRandom randomGenerator = ThreadLocalRandom.current();
-      String tableSpec = tableSpecFromWindowedValue(
-          context.getPipelineOptions().as(BigQueryOptions.class),
-          ValueInSingleWindow.of(context.element(), context.timestamp(), window, context.pane()));
-      // We output on keys 0-50 to ensure that there's enough batching for
-      // BigQuery.
-      context.output(KV.of(ShardedKey.of(tableSpec, randomGenerator.nextInt(0, 50)),
-          new TableRowInfo(formatFunction.apply(context.element()), uniqueId)));
-    }
-
-    @Override
-    public void populateDisplayData(DisplayData.Builder builder) {
-      super.populateDisplayData(builder);
-
-      builder.addIfNotNull(DisplayData.item("table", tableSpec));
-      if (tableRefFunction != null) {
-        builder.add(DisplayData.item("tableFn", tableRefFunction.getClass())
-          .withLabel("Table Reference Function"));
-      }
-    }
-
-    @VisibleForTesting
-    ValueProvider<String> getTableSpec() {
-      return tableSpec;
-    }
-
-    private String tableSpecFromWindowedValue(BigQueryOptions options,
-                                              ValueInSingleWindow<T> value) {
-      if (tableSpec != null) {
-        return tableSpec.get();
-      } else {
-        TableReference table = tableRefFunction.apply(value);
-        if (table.getProjectId() == null) {
-          table.setProjectId(options.getProject());
-        }
-        return BigQueryHelpers.toTableSpec(table);
-      }
-    }
-  }
-
-  /////////////////////////////////////////////////////////////////////////////
-
-  /**
-  * PTransform that performs streaming BigQuery write. To increase consistency,
-  * it leverages BigQuery best effort de-dup mechanism.
-   */
-  private static class StreamWithDeDup<T> extends PTransform<PCollection<T>, PDone> {
-    private final Write<T> write;
-
-    /** Constructor. */
-    StreamWithDeDup(Write<T> write) {
-      this.write = write;
-    }
-
-    @Override
-    protected Coder<Void> getDefaultOutputCoder() {
-      return VoidCoder.of();
-    }
-
-    @Override
-    public PDone expand(PCollection<T> input) {
-      // A naive implementation would be to simply stream data directly to BigQuery.
-      // However, this could occasionally lead to duplicated data, e.g., when
-      // a VM that runs this code is restarted and the code is re-run.
-
-      // The above risk is mitigated in this implementation by relying on
-      // BigQuery built-in best effort de-dup mechanism.
-
-      // To use this mechanism, each input TableRow is tagged with a generated
-      // unique id, which is then passed to BigQuery and used to ignore duplicates.
-
-      PCollection<KV<ShardedKey<String>, TableRowInfo>> tagged =
-          input.apply(ParDo.of(new TagWithUniqueIdsAndTable<T>(
-              input.getPipeline().getOptions().as(BigQueryOptions.class), write.getTable(),
-              write.getTableRefFunction(), write.getFormatFunction())));
-
-      // To prevent having the same TableRow processed more than once with regenerated
-      // different unique ids, this implementation relies on "checkpointing", which is
-      // achieved as a side effect of having StreamingWriteFn immediately follow a GBK,
-      // performed by Reshuffle.
-      NestedValueProvider<TableSchema, String> schema =
-          write.getJsonSchema() == null
-              ? null
-              : NestedValueProvider.of(write.getJsonSchema(), new JsonSchemaToTableSchema());
-      tagged
-          .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowInfoCoder.of()))
-          .apply(Reshuffle.<ShardedKey<String>, TableRowInfo>of())
-          .apply(
-              ParDo.of(
-                  new StreamingWriteFn(
-                      schema,
-                      write.getCreateDisposition(),
-                      write.getTableDescription(),
-                      write.getBigQueryServices())));
-
-      // Note that the implementation to return PDone here breaks the
-      // implicit assumption about the job execution order. If a user
-      // implements a PTransform that takes PDone returned here as its
-      // input, the transform may not necessarily be executed after
-      // the BigQueryIO.Write.
-
-      return PDone.in(input.getPipeline());
-    }
-  }
-
-  /**
-   * Status of a BigQuery job or request.
-   */
-  enum Status {
-    SUCCEEDED,
-    FAILED,
-    UNKNOWN,
-  }
 
   /////////////////////////////////////////////////////////////////////////////
 

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
index a909957..9153157 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.beam.sdk.io.gcp.bigquery;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -14,9 +32,10 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToProjectId;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
 import org.apache.beam.sdk.options.BigQueryOptions;

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index ff50e6d..746258f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.beam.sdk.io.gcp.bigquery;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -18,7 +36,7 @@ import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.TableRowJsonCoder;
 import org.apache.beam.sdk.io.AvroSource;
 import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
 import org.apache.beam.sdk.options.BigQueryOptions;
 import org.apache.beam.sdk.options.PipelineOptions;

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
index aae0faa..cbd5781 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.beam.sdk.io.gcp.bigquery;
 
 import static com.google.common.base.Preconditions.checkNotNull;

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
index 612afbe..75f7b93 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.beam.sdk.io.gcp.bigquery;
 
 import com.google.common.annotations.VisibleForTesting;

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
new file mode 100644
index 0000000..8c968df
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+/**
+ * A key and a shard number.
+ */
+class ShardedKey<K> {
+  private final K key;
+  private final int shardNumber;
+
+  public static <K> ShardedKey<K> of(K key, int shardNumber) {
+    return new ShardedKey<>(key, shardNumber);
+  }
+
+  ShardedKey(K key, int shardNumber) {
+    this.key = key;
+    this.shardNumber = shardNumber;
+  }
+
+  public K getKey() {
+    return key;
+  }
+
+  public int getShardNumber() {
+    return shardNumber;
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
new file mode 100644
index 0000000..be4e71c
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StandardCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.util.PropertyNames;
+
+
+/**
+ * A {@link Coder} for {@link ShardedKey}, using a wrapped key {@link Coder}.
+ */
+@VisibleForTesting
+class ShardedKeyCoder<KeyT>
+    extends StandardCoder<ShardedKey<KeyT>> {
+  public static <KeyT> ShardedKeyCoder<KeyT> of(Coder<KeyT> keyCoder) {
+    return new ShardedKeyCoder<>(keyCoder);
+  }
+
+  @JsonCreator
+  public static <KeyT> ShardedKeyCoder<KeyT> of(
+       @JsonProperty(PropertyNames.COMPONENT_ENCODINGS)
+           List<Coder<KeyT>> components) {
+    checkArgument(components.size() == 1, "Expecting 1 component, got %s", components.size());
+    return of(components.get(0));
+  }
+
+  protected ShardedKeyCoder(Coder<KeyT> keyCoder) {
+    this.keyCoder = keyCoder;
+    this.shardNumberCoder = VarIntCoder.of();
+  }
+
+  @Override
+  public List<? extends Coder<?>> getCoderArguments() {
+    return Arrays.asList(keyCoder);
+  }
+
+  @Override
+  public void encode(ShardedKey<KeyT> key, OutputStream outStream, Context context)
+      throws IOException {
+    keyCoder.encode(key.getKey(), outStream, context.nested());
+    shardNumberCoder.encode(key.getShardNumber(), outStream, context);
+  }
+
+  @Override
+  public ShardedKey<KeyT> decode(InputStream inStream, Context context)
+      throws IOException {
+    return new ShardedKey<>(
+        keyCoder.decode(inStream, context.nested()),
+        shardNumberCoder.decode(inStream, context));
+  }
+
+  @Override
+  public void verifyDeterministic() throws NonDeterministicException {
+    keyCoder.verifyDeterministic();
+  }
+
+  Coder<KeyT> keyCoder;
+  VarIntCoder shardNumberCoder;
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java
new file mode 100644
index 0000000..f667295
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableSchema;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.Reshuffle;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+
+/**
+* PTransform that performs streaming BigQuery write. To increase consistency,
+* it leverages BigQuery best effort de-dup mechanism.
+ */
+class StreamWithDeDup<T> extends PTransform<PCollection<T>, PDone> {
+  private final Write<T> write;
+
+  /** Constructor. */
+  StreamWithDeDup(Write<T> write) {
+    this.write = write;
+  }
+
+  @Override
+  protected Coder<Void> getDefaultOutputCoder() {
+    return VoidCoder.of();
+  }
+
+  @Override
+  public PDone expand(PCollection<T> input) {
+    // A naive implementation would be to simply stream data directly to BigQuery.
+    // However, this could occasionally lead to duplicated data, e.g., when
+    // a VM that runs this code is restarted and the code is re-run.
+
+    // The above risk is mitigated in this implementation by relying on
+    // BigQuery built-in best effort de-dup mechanism.
+
+    // To use this mechanism, each input TableRow is tagged with a generated
+    // unique id, which is then passed to BigQuery and used to ignore duplicates.
+
+    PCollection<KV<ShardedKey<String>, TableRowInfo>> tagged =
+        input.apply(ParDo.of(new TagWithUniqueIdsAndTable<T>(
+            input.getPipeline().getOptions().as(BigQueryOptions.class), write.getTable(),
+            write.getTableRefFunction(), write.getFormatFunction())));
+
+    // To prevent having the same TableRow processed more than once with regenerated
+    // different unique ids, this implementation relies on "checkpointing", which is
+    // achieved as a side effect of having StreamingWriteFn immediately follow a GBK,
+    // performed by Reshuffle.
+    NestedValueProvider<TableSchema, String> schema =
+        write.getJsonSchema() == null
+            ? null
+            : NestedValueProvider.of(write.getJsonSchema(), new JsonSchemaToTableSchema());
+    tagged
+        .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowInfoCoder.of()))
+        .apply(Reshuffle.<ShardedKey<String>, TableRowInfo>of())
+        .apply(
+            ParDo.of(
+                new StreamingWriteFn(
+                    schema,
+                    write.getCreateDisposition(),
+                    write.getTableDescription(),
+                    write.getBigQueryServices())));
+
+    // Note that the implementation to return PDone here breaks the
+    // implicit assumption about the job execution order. If a user
+    // implements a PTransform that takes PDone returned here as its
+    // input, the transform may not necessarily be executed after
+    // the BigQueryIO.Write.
+
+    return PDone.in(input.getPipeline());
+  }
+}


[4/6] beam git commit: Refactor BigQueryIO helper methods into a BigQueryHelpers class. Move helper transforms for BigQueryIO.Read into individual files. Change private to package in BigQueryHelpers.

Posted by jk...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 35bb9c3..eca95b9 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -20,8 +20,8 @@ package org.apache.beam.sdk.io.gcp.bigquery;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.fromJsonString;
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.toJsonString;
+import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.fromJsonString;
+import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.toJsonString;
 import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.hasItem;
@@ -101,13 +101,9 @@ import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.CountingInput;
 import org.apache.beam.sdk.io.CountingSource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.BigQueryQuerySource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.BigQueryTableSource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.JsonSchemaToTableSchema;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.PassThroughThenCleanup;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.PassThroughThenCleanup.CleanupOperation;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TransformingSource;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.TableRowWriter;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
@@ -116,6 +112,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteRename;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteTables;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.io.gcp.bigquery.PassThroughThenCleanup.CleanupOperation;
 import org.apache.beam.sdk.options.BigQueryOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -1582,7 +1579,7 @@ public class BigQueryIOTest implements Serializable {
   public void testStreamingWriteFnCreateNever() throws Exception {
     BigQueryIO.StreamingWriteFn fn = new BigQueryIO.StreamingWriteFn(
         null, CreateDisposition.CREATE_NEVER, null, new FakeBigQueryServices());
-    assertEquals(BigQueryIO.parseTableSpec("dataset.table"),
+    assertEquals(BigQueryHelpers.parseTableSpec("dataset.table"),
         fn.getOrCreateTable(null, "dataset.table"));
   }
 
@@ -1616,7 +1613,7 @@ public class BigQueryIOTest implements Serializable {
 
   @Test
   public void testTableParsing() {
-    TableReference ref = BigQueryIO
+    TableReference ref = BigQueryHelpers
         .parseTableSpec("my-project:data_set.table_name");
     Assert.assertEquals("my-project", ref.getProjectId());
     Assert.assertEquals("data_set", ref.getDatasetId());
@@ -1625,14 +1622,14 @@ public class BigQueryIOTest implements Serializable {
 
   @Test
   public void testTableParsing_validPatterns() {
-    BigQueryIO.parseTableSpec("a123-456:foo_bar.d");
-    BigQueryIO.parseTableSpec("a12345:b.c");
-    BigQueryIO.parseTableSpec("b12345.c");
+    BigQueryHelpers.parseTableSpec("a123-456:foo_bar.d");
+    BigQueryHelpers.parseTableSpec("a12345:b.c");
+    BigQueryHelpers.parseTableSpec("b12345.c");
   }
 
   @Test
   public void testTableParsing_noProjectId() {
-    TableReference ref = BigQueryIO
+    TableReference ref = BigQueryHelpers
         .parseTableSpec("data_set.table_name");
     Assert.assertEquals(null, ref.getProjectId());
     Assert.assertEquals("data_set", ref.getDatasetId());
@@ -1642,25 +1639,25 @@ public class BigQueryIOTest implements Serializable {
   @Test
   public void testTableParsingError() {
     thrown.expect(IllegalArgumentException.class);
-    BigQueryIO.parseTableSpec("0123456:foo.bar");
+    BigQueryHelpers.parseTableSpec("0123456:foo.bar");
   }
 
   @Test
   public void testTableParsingError_2() {
     thrown.expect(IllegalArgumentException.class);
-    BigQueryIO.parseTableSpec("myproject:.bar");
+    BigQueryHelpers.parseTableSpec("myproject:.bar");
   }
 
   @Test
   public void testTableParsingError_3() {
     thrown.expect(IllegalArgumentException.class);
-    BigQueryIO.parseTableSpec(":a.b");
+    BigQueryHelpers.parseTableSpec(":a.b");
   }
 
   @Test
   public void testTableParsingError_slash() {
     thrown.expect(IllegalArgumentException.class);
-    BigQueryIO.parseTableSpec("a\\b12345:c.d");
+    BigQueryHelpers.parseTableSpec("a\\b12345:c.d");
   }
 
   // Test that BigQuery's special null placeholder objects can be encoded.
@@ -1709,7 +1706,7 @@ public class BigQueryIOTest implements Serializable {
             toJsonString(new TableRow().set("name", "c").set("number", "3")));
 
     String jobIdToken = "testJobIdToken";
-    TableReference table = BigQueryIO.parseTableSpec("project.data_set.table_name");
+    TableReference table = BigQueryHelpers.parseTableSpec("project.data_set.table_name");
     String extractDestinationDir = "mock://tempLocation";
     BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
         StaticValueProvider.of(jobIdToken), StaticValueProvider.of(table),
@@ -1748,7 +1745,7 @@ public class BigQueryIOTest implements Serializable {
             toJsonString(new TableRow().set("name", "c").set("number", "3")));
 
     String jobIdToken = "testJobIdToken";
-    TableReference table = BigQueryIO.parseTableSpec("project:data_set.table_name");
+    TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
     String extractDestinationDir = "mock://tempLocation";
     BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
         StaticValueProvider.of(jobIdToken), StaticValueProvider.of(table),
@@ -1815,7 +1812,7 @@ public class BigQueryIOTest implements Serializable {
 
     String jobIdToken = "testJobIdToken";
     String extractDestinationDir = "mock://tempLocation";
-    TableReference destinationTable = BigQueryIO.parseTableSpec("project:data_set.table_name");
+    TableReference destinationTable = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
     BoundedSource<TableRow> bqSource = BigQueryQuerySource.create(
         StaticValueProvider.of(jobIdToken), StaticValueProvider.of("query"),
         StaticValueProvider.of(destinationTable),
@@ -1904,7 +1901,7 @@ public class BigQueryIOTest implements Serializable {
 
     String jobIdToken = "testJobIdToken";
     String extractDestinationDir = "mock://tempLocation";
-    TableReference destinationTable = BigQueryIO.parseTableSpec("project:data_set.table_name");
+    TableReference destinationTable = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
     BoundedSource<TableRow> bqSource = BigQueryQuerySource.create(
         StaticValueProvider.of(jobIdToken), StaticValueProvider.of("query"),
         StaticValueProvider.of(destinationTable),
@@ -2285,9 +2282,12 @@ public class BigQueryIOTest implements Serializable {
     String datasetId = "somedataset";
     List<String> tables = Lists.newArrayList("table1", "table2", "table3");
     List<TableReference> tableRefs = Lists.newArrayList(
-        BigQueryIO.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, tables.get(0))),
-        BigQueryIO.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, tables.get(1))),
-        BigQueryIO.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, tables.get(2))));
+        BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId,
+            tables.get(0))),
+        BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId,
+            tables.get(1))),
+        BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId,
+            tables.get(2))));
 
     doThrow(new IOException("Unable to delete table"))
         .when(mockDatasetService).deleteTable(tableRefs.get(0));
@@ -2371,8 +2371,8 @@ public class BigQueryIOTest implements Serializable {
         new BigQueryIO.TagWithUniqueIdsAndTable<TableRow>(
             bqOptions, NestedValueProvider.of(
                 StaticValueProvider.of("data_set.table_name"),
-                new BigQueryIO.TableSpecToTableRef()), null, null);
-    TableReference table = BigQueryIO.parseTableSpec(tag.getTableSpec().get());
+                new TableSpecToTableRef()), null, null);
+    TableReference table = BigQueryHelpers.parseTableSpec(tag.getTableSpec().get());
     assertNotNull(table.getProjectId());
   }
 

http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
index 7b5b226..fa84119 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
@@ -222,7 +222,7 @@ public class BigQueryUtilTest {
     List<TableRow> rows = new ArrayList<>();
     try (BigQueryTableRowIterator iterator =
             BigQueryTableRowIterator.fromTable(
-                BigQueryIO.parseTableSpec("project:dataset.table"), mockClient)) {
+                BigQueryHelpers.parseTableSpec("project:dataset.table"), mockClient)) {
       iterator.open();
       while (iterator.advance()) {
         rows.add(iterator.getCurrent());
@@ -261,7 +261,7 @@ public class BigQueryUtilTest {
     onTableList(dataList);
 
     try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
-        BigQueryIO.parseTableSpec("project:dataset.table"),
+        BigQueryHelpers.parseTableSpec("project:dataset.table"),
         mockClient)) {
       iterator.open();
       Assert.assertTrue(iterator.advance());
@@ -291,7 +291,7 @@ public class BigQueryUtilTest {
     onTableList(dataList);
 
     try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
-        BigQueryIO.parseTableSpec("project:dataset.table"),
+        BigQueryHelpers.parseTableSpec("project:dataset.table"),
         mockClient)) {
       iterator.open();
 
@@ -320,7 +320,7 @@ public class BigQueryUtilTest {
         .thenReturn(page2);
 
     try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
-        BigQueryIO.parseTableSpec("project:dataset.table"),
+        BigQueryHelpers.parseTableSpec("project:dataset.table"),
         mockClient)) {
       iterator.open();
 
@@ -350,7 +350,7 @@ public class BigQueryUtilTest {
         .thenThrow(new IOException("No such table"));
 
     try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
-        BigQueryIO.parseTableSpec("project:dataset.table"),
+        BigQueryHelpers.parseTableSpec("project:dataset.table"),
         mockClient)) {
       try {
         iterator.open(); // throws.
@@ -387,7 +387,7 @@ public class BigQueryUtilTest {
     errorsIndices.add(new ArrayList<Long>());
     onInsertAll(errorsIndices);
 
-    TableReference ref = BigQueryIO
+    TableReference ref = BigQueryHelpers
         .parseTableSpec("project:dataset.table");
     DatasetServiceImpl datasetService = new DatasetServiceImpl(mockClient, options, 5);
 


[2/6] beam git commit: Refactor BigQueryIO.Write helper transforms into separate files.

Posted by jk...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
new file mode 100644
index 0000000..1d93fa3
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.SystemDoFnInternal;
+import org.apache.beam.sdk.values.KV;
+
+/**
+ * Implementation of DoFn to perform streaming BigQuery write.
+ */
+@SystemDoFnInternal
+@VisibleForTesting
+class StreamingWriteFn
+    extends DoFn<KV<ShardedKey<String>, TableRowInfo>, Void> {
+  /** TableSchema in JSON. Use String to make the class Serializable. */
+  @Nullable
+  private final ValueProvider<String> jsonTableSchema;
+
+  @Nullable private final String tableDescription;
+
+  private final BigQueryServices bqServices;
+
+  /** JsonTableRows to accumulate BigQuery rows in order to batch writes. */
+  private transient Map<String, List<TableRow>> tableRows;
+
+  private final Write.CreateDisposition createDisposition;
+
+  /** The list of unique ids for each BigQuery table row. */
+  private transient Map<String, List<String>> uniqueIdsForTableRows;
+
+  /** The list of tables created so far, so we don't try the creation
+      each time. */
+  private static Set<String> createdTables =
+      Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+
+  /** Tracks bytes written, exposed as "ByteCount" Counter. */
+  private Counter byteCounter = Metrics.counter(StreamingWriteFn.class, "ByteCount");
+
+  /** Constructor. */
+  StreamingWriteFn(@Nullable ValueProvider<TableSchema> schema,
+                   Write.CreateDisposition createDisposition,
+                   @Nullable String tableDescription, BigQueryServices bqServices) {
+    this.jsonTableSchema = schema == null ? null :
+        NestedValueProvider.of(schema, new TableSchemaToJsonSchema());
+    this.createDisposition = createDisposition;
+    this.bqServices = checkNotNull(bqServices, "bqServices");
+    this.tableDescription = tableDescription;
+  }
+
+  /**
+   * Clear the cached map of created tables. Used for testing.
+   */
+  static void clearCreatedTables() {
+    synchronized (createdTables) {
+      createdTables.clear();
+    }
+  }
+
+  /** Prepares a target BigQuery table. */
+  @StartBundle
+  public void startBundle(Context context) {
+    tableRows = new HashMap<>();
+    uniqueIdsForTableRows = new HashMap<>();
+  }
+
+  /** Accumulates the input into JsonTableRows and uniqueIdsForTableRows. */
+  @ProcessElement
+  public void processElement(ProcessContext context) {
+    String tableSpec = context.element().getKey().getKey();
+    List<TableRow> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
+    List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows,
+        tableSpec);
+
+    rows.add(context.element().getValue().tableRow);
+    uniqueIds.add(context.element().getValue().uniqueId);
+  }
+
+  /** Writes the accumulated rows into BigQuery with streaming API. */
+  @FinishBundle
+  public void finishBundle(Context context) throws Exception {
+    BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class);
+
+    for (Map.Entry<String, List<TableRow>> entry : tableRows.entrySet()) {
+      TableReference tableReference = getOrCreateTable(options, entry.getKey());
+      flushRows(tableReference, entry.getValue(),
+          uniqueIdsForTableRows.get(entry.getKey()), options);
+    }
+    tableRows.clear();
+    uniqueIdsForTableRows.clear();
+  }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+
+    builder
+        .addIfNotNull(DisplayData.item("schema", jsonTableSchema)
+          .withLabel("Table Schema"))
+        .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
+          .withLabel("Table Description"));
+  }
+
+  public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
+      throws InterruptedException, IOException {
+    TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec);
+    if (createDisposition != createDisposition.CREATE_NEVER
+        && !createdTables.contains(tableSpec)) {
+      synchronized (createdTables) {
+        // Another thread may have succeeded in creating the table in the meanwhile, so
+        // check again. This check isn't needed for correctness, but we add it to prevent
+        // every thread from attempting a create and overwhelming our BigQuery quota.
+        DatasetService datasetService = bqServices.getDatasetService(options);
+        if (!createdTables.contains(tableSpec)) {
+          if (datasetService.getTable(tableReference) == null) {
+            TableSchema tableSchema = BigQueryIO.JSON_FACTORY.fromString(
+                jsonTableSchema.get(), TableSchema.class);
+            datasetService.createTable(
+                new Table()
+                    .setTableReference(tableReference)
+                    .setSchema(tableSchema)
+                    .setDescription(tableDescription));
+          }
+          createdTables.add(tableSpec);
+        }
+      }
+    }
+    return tableReference;
+  }
+
+  /**
+   * Writes the accumulated rows into BigQuery with streaming API.
+   */
+  private void flushRows(TableReference tableReference,
+      List<TableRow> tableRows, List<String> uniqueIds, BigQueryOptions options)
+          throws InterruptedException {
+    if (!tableRows.isEmpty()) {
+      try {
+        long totalBytes = bqServices.getDatasetService(options).insertAll(
+            tableReference, tableRows, uniqueIds);
+        byteCounter.inc(totalBytes);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java
new file mode 100644
index 0000000..4a59ab6
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableRow;
+
+/**
+ * Encapsulates a {@link TableRow} along with a unique insertion id.
+ */
+class TableRowInfo {
+  TableRowInfo(TableRow tableRow, String uniqueId) {
+    this.tableRow = tableRow;
+    this.uniqueId = uniqueId;
+  }
+
+  final TableRow tableRow;
+  final String uniqueId;
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
new file mode 100644
index 0000000..5e8fa29
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.TableRowJsonCoder;
+
+/**
+ * Defines a coder for {@link TableRowInfo} objects.
+ */
+@VisibleForTesting
+class TableRowInfoCoder extends AtomicCoder<TableRowInfo> {
+  private static final TableRowInfoCoder INSTANCE = new TableRowInfoCoder();
+
+  @JsonCreator
+  public static TableRowInfoCoder of() {
+    return INSTANCE;
+  }
+
+  @Override
+  public void encode(TableRowInfo value, OutputStream outStream, Context context)
+      throws IOException {
+    if (value == null) {
+      throw new CoderException("cannot encode a null value");
+    }
+    tableRowCoder.encode(value.tableRow, outStream, context.nested());
+    idCoder.encode(value.uniqueId, outStream, context);
+  }
+
+  @Override
+  public TableRowInfo decode(InputStream inStream, Context context)
+      throws IOException {
+    return new TableRowInfo(
+        tableRowCoder.decode(inStream, context.nested()),
+        idCoder.decode(inStream, context));
+  }
+
+  @Override
+  public void verifyDeterministic() throws NonDeterministicException {
+    throw new NonDeterministicException(this, "TableRows are not deterministic.");
+  }
+
+  TableRowJsonCoder tableRowCoder = TableRowJsonCoder.of();
+  StringUtf8Coder idCoder = StringUtf8Coder.of();
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
new file mode 100644
index 0000000..014c498
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.io.CountingOutputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.nio.charset.StandardCharsets;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.Coder.Context;
+import org.apache.beam.sdk.coders.TableRowJsonCoder;
+import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.MimeTypes;
+import org.apache.beam.sdk.values.KV;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Writes {@TableRow} objects out to a file. Used when doing batch load jobs into BigQuery.
+ */
+class TableRowWriter {
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryIO.class);
+
+  private static final Coder<TableRow> CODER = TableRowJsonCoder.of();
+  private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
+  private final String tempFilePrefix;
+  private String id;
+  private String fileName;
+  private WritableByteChannel channel;
+  protected String mimeType = MimeTypes.TEXT;
+  private CountingOutputStream out;
+
+  TableRowWriter(String basename) {
+    this.tempFilePrefix = basename;
+  }
+
+  public final void open(String uId) throws Exception {
+    id = uId;
+    fileName = tempFilePrefix + id;
+    LOG.debug("Opening {}.", fileName);
+    channel = IOChannelUtils.create(fileName, mimeType);
+    try {
+      out = new CountingOutputStream(Channels.newOutputStream(channel));
+      LOG.debug("Writing header to {}.", fileName);
+    } catch (Exception e) {
+      try {
+        LOG.error("Writing header to {} failed, closing channel.", fileName);
+        channel.close();
+      } catch (IOException closeException) {
+        LOG.error("Closing channel for {} failed", fileName);
+      }
+      throw e;
+    }
+    LOG.debug("Starting write of bundle {} to {}.", this.id, fileName);
+  }
+
+  public void write(TableRow value) throws Exception {
+    CODER.encode(value, out, Context.OUTER);
+    out.write(NEWLINE);
+  }
+
+  public final KV<String, Long> close() throws IOException {
+    channel.close();
+    return KV.of(fileName, out.getCount());
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java
new file mode 100644
index 0000000..a6608e4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonTableRefToTableRef;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToTableSpec;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
+
+/**
+ * Fn that tags each table row with a unique id and destination table.
+ * To avoid calling UUID.randomUUID() for each element, which can be costly,
+ * a randomUUID is generated only once per bucket of data. The actual unique
+ * id is created by concatenating this randomUUID with a sequential number.
+ */
+@VisibleForTesting
+class TagWithUniqueIdsAndTable<T>
+    extends DoFn<T, KV<ShardedKey<String>, TableRowInfo>> {
+  /** TableSpec to write to. */
+  private final ValueProvider<String> tableSpec;
+
+  /** User function mapping windowed values to {@link TableReference} in JSON. */
+  private final SerializableFunction<ValueInSingleWindow<T>, TableReference> tableRefFunction;
+
+  /** User function mapping user type to a TableRow. */
+  private final SerializableFunction<T, TableRow> formatFunction;
+
+  private transient String randomUUID;
+  private transient long sequenceNo = 0L;
+
+  TagWithUniqueIdsAndTable(BigQueryOptions options,
+                           ValueProvider<TableReference> table,
+                           SerializableFunction<ValueInSingleWindow<T>, TableReference>
+                               tableRefFunction,
+                           SerializableFunction<T, TableRow> formatFunction) {
+    checkArgument(table == null ^ tableRefFunction == null,
+        "Exactly one of table or tableRefFunction should be set");
+    if (table != null) {
+      if (table.isAccessible() && Strings.isNullOrEmpty(table.get().getProjectId())) {
+        TableReference tableRef = table.get()
+            .setProjectId(options.as(BigQueryOptions.class).getProject());
+        table = NestedValueProvider.of(
+            StaticValueProvider.of(BigQueryHelpers.toJsonString(tableRef)),
+            new JsonTableRefToTableRef());
+      }
+      this.tableSpec = NestedValueProvider.of(table, new TableRefToTableSpec());
+    } else {
+      tableSpec = null;
+    }
+    this.tableRefFunction = tableRefFunction;
+    this.formatFunction = formatFunction;
+  }
+
+
+  @StartBundle
+  public void startBundle(Context context) {
+    randomUUID = UUID.randomUUID().toString();
+  }
+
+  /** Tag the input with a unique id. */
+  @ProcessElement
+  public void processElement(ProcessContext context, BoundedWindow window) throws IOException {
+    String uniqueId = randomUUID + sequenceNo++;
+    ThreadLocalRandom randomGenerator = ThreadLocalRandom.current();
+    String tableSpec = tableSpecFromWindowedValue(
+        context.getPipelineOptions().as(BigQueryOptions.class),
+        ValueInSingleWindow.of(context.element(), context.timestamp(), window, context.pane()));
+    // We output on keys 0-50 to ensure that there's enough batching for
+    // BigQuery.
+    context.output(KV.of(ShardedKey.of(tableSpec, randomGenerator.nextInt(0, 50)),
+        new TableRowInfo(formatFunction.apply(context.element()), uniqueId)));
+  }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+
+    builder.addIfNotNull(DisplayData.item("table", tableSpec));
+    if (tableRefFunction != null) {
+      builder.add(DisplayData.item("tableFn", tableRefFunction.getClass())
+        .withLabel("Table Reference Function"));
+    }
+  }
+
+  @VisibleForTesting
+  ValueProvider<String> getTableSpec() {
+    return tableSpec;
+  }
+
+  private String tableSpecFromWindowedValue(BigQueryOptions options,
+                                            ValueInSingleWindow<T> value) {
+    if (tableSpec != null) {
+      return tableSpec.get();
+    } else {
+      TableReference table = tableRefFunction.apply(value);
+      if (table.getProjectId() == null) {
+        table.setProjectId(options.getProject());
+      }
+      return BigQueryHelpers.toTableSpec(table);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
index a86adfb..f7d8252 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.beam.sdk.io.gcp.bigquery;
 
 import static com.google.common.base.Preconditions.checkNotNull;

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java
new file mode 100644
index 0000000..6219226
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableRow;
+import java.util.UUID;
+
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.KV;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Writes each bundle of {@link TableRow} elements out to a separate file using
+ * {@link TableRowWriter}.
+ */
+class WriteBundles extends DoFn<TableRow, KV<String, Long>> {
+  private static final Logger LOG = LoggerFactory.getLogger(WriteBundles.class);
+
+  private transient TableRowWriter writer = null;
+  private final String tempFilePrefix;
+
+  WriteBundles(String tempFilePrefix) {
+    this.tempFilePrefix = tempFilePrefix;
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) throws Exception {
+    if (writer == null) {
+      writer = new TableRowWriter(tempFilePrefix);
+      writer.open(UUID.randomUUID().toString());
+      LOG.debug("Done opening writer {}", writer);
+    }
+    try {
+      writer.write(c.element());
+    } catch (Exception e) {
+      // Discard write result and close the write.
+      try {
+        writer.close();
+        // The writer does not need to be reset, as this DoFn cannot be reused.
+      } catch (Exception closeException) {
+        // Do not mask the exception that caused the write to fail.
+        e.addSuppressed(closeException);
+      }
+      throw e;
+    }
+  }
+
+  @FinishBundle
+  public void finishBundle(Context c) throws Exception {
+    if (writer != null) {
+      c.output(writer.close());
+      writer = null;
+    }
+  }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+
+    builder
+        .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
+            .withLabel("Temporary File Prefix"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
new file mode 100644
index 0000000..f4bf198
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.common.collect.Lists;
+import java.util.List;
+import java.util.UUID;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Partitions temporary files based on number of files and file sizes.
+ */
+class WritePartition extends DoFn<String, KV<Long, List<String>>> {
+  private final PCollectionView<Iterable<KV<String, Long>>> resultsView;
+  private TupleTag<KV<Long, List<String>>> multiPartitionsTag;
+  private TupleTag<KV<Long, List<String>>> singlePartitionTag;
+
+  public WritePartition(
+      PCollectionView<Iterable<KV<String, Long>>> resultsView,
+      TupleTag<KV<Long, List<String>>> multiPartitionsTag,
+      TupleTag<KV<Long, List<String>>> singlePartitionTag) {
+    this.resultsView = resultsView;
+    this.multiPartitionsTag = multiPartitionsTag;
+    this.singlePartitionTag = singlePartitionTag;
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) throws Exception {
+    List<KV<String, Long>> results = Lists.newArrayList(c.sideInput(resultsView));
+    if (results.isEmpty()) {
+      TableRowWriter writer = new TableRowWriter(c.element());
+      writer.open(UUID.randomUUID().toString());
+      results.add(writer.close());
+    }
+
+    long partitionId = 0;
+    int currNumFiles = 0;
+    long currSizeBytes = 0;
+    List<String> currResults = Lists.newArrayList();
+    for (int i = 0; i < results.size(); ++i) {
+      KV<String, Long> fileResult = results.get(i);
+      if (currNumFiles + 1 > Write.MAX_NUM_FILES
+          || currSizeBytes + fileResult.getValue() > Write.MAX_SIZE_BYTES) {
+        c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
+        currResults = Lists.newArrayList();
+        currNumFiles = 0;
+        currSizeBytes = 0;
+      }
+      ++currNumFiles;
+      currSizeBytes += fileResult.getValue();
+      currResults.add(fileResult.getKey());
+    }
+    if (partitionId == 0) {
+      c.sideOutput(singlePartitionTag, KV.of(++partitionId, currResults));
+    } else {
+      c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
new file mode 100644
index 0000000..8cb9439
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Copies temporary tables to destination table.
+ */
+class WriteRename extends DoFn<String, Void> {
+  private static final Logger LOG = LoggerFactory.getLogger(WriteRename.class);
+
+  private final BigQueryServices bqServices;
+  private final PCollectionView<String> jobIdToken;
+  private final ValueProvider<String> jsonTableRef;
+  private final WriteDisposition writeDisposition;
+  private final CreateDisposition createDisposition;
+  private final PCollectionView<Iterable<String>> tempTablesView;
+  @Nullable
+  private final String tableDescription;
+
+  public WriteRename(
+      BigQueryServices bqServices,
+      PCollectionView<String> jobIdToken,
+      ValueProvider<String> jsonTableRef,
+      WriteDisposition writeDisposition,
+      CreateDisposition createDisposition,
+      PCollectionView<Iterable<String>> tempTablesView,
+      @Nullable String tableDescription) {
+    this.bqServices = bqServices;
+    this.jobIdToken = jobIdToken;
+    this.jsonTableRef = jsonTableRef;
+    this.writeDisposition = writeDisposition;
+    this.createDisposition = createDisposition;
+    this.tempTablesView = tempTablesView;
+    this.tableDescription = tableDescription;
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) throws Exception {
+    List<String> tempTablesJson = Lists.newArrayList(c.sideInput(tempTablesView));
+
+    // Do not copy if no temp tables are provided
+    if (tempTablesJson.size() == 0) {
+      return;
+    }
+
+    List<TableReference> tempTables = Lists.newArrayList();
+    for (String table : tempTablesJson) {
+      tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
+    }
+    copy(
+        bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
+        bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
+        c.sideInput(jobIdToken),
+        BigQueryHelpers.fromJsonString(jsonTableRef.get(), TableReference.class),
+        tempTables,
+        writeDisposition,
+        createDisposition,
+        tableDescription);
+
+    DatasetService tableService =
+        bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class));
+    removeTemporaryTables(tableService, tempTables);
+  }
+
+  private void copy(
+      JobService jobService,
+      DatasetService datasetService,
+      String jobIdPrefix,
+      TableReference ref,
+      List<TableReference> tempTables,
+      WriteDisposition writeDisposition,
+      CreateDisposition createDisposition,
+      @Nullable String tableDescription) throws InterruptedException, IOException {
+    JobConfigurationTableCopy copyConfig = new JobConfigurationTableCopy()
+        .setSourceTables(tempTables)
+        .setDestinationTable(ref)
+        .setWriteDisposition(writeDisposition.name())
+        .setCreateDisposition(createDisposition.name());
+
+    String projectId = ref.getProjectId();
+    Job lastFailedCopyJob = null;
+    for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
+      String jobId = jobIdPrefix + "-" + i;
+      JobReference jobRef = new JobReference()
+          .setProjectId(projectId)
+          .setJobId(jobId);
+      jobService.startCopyJob(jobRef, copyConfig);
+      Job copyJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
+      Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
+      switch (jobStatus) {
+        case SUCCEEDED:
+          if (tableDescription != null) {
+            datasetService.patchTableDescription(ref, tableDescription);
+          }
+          return;
+        case UNKNOWN:
+          throw new RuntimeException(String.format(
+              "UNKNOWN status of copy job [%s]: %s.", jobId,
+              BigQueryHelpers.jobToPrettyString(copyJob)));
+        case FAILED:
+          lastFailedCopyJob = copyJob;
+          continue;
+        default:
+          throw new IllegalStateException(String.format(
+              "Unexpected status [%s] of load job: %s.",
+              jobStatus, BigQueryHelpers.jobToPrettyString(copyJob)));
+      }
+    }
+    throw new RuntimeException(String.format(
+        "Failed to create copy job with id prefix %s, "
+            + "reached max retries: %d, last failed copy job: %s.",
+        jobIdPrefix,
+        Write.MAX_RETRY_JOBS,
+        BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
+  }
+
+  static void removeTemporaryTables(DatasetService tableService,
+      List<TableReference> tempTables) {
+    for (TableReference tableRef : tempTables) {
+      try {
+        LOG.debug("Deleting table {}", BigQueryHelpers.toJsonString(tableRef));
+        tableService.deleteTable(tableRef);
+      } catch (Exception e) {
+        LOG.warn("Failed to delete the table {}", BigQueryHelpers.toJsonString(tableRef), e);
+      }
+    }
+  }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+
+    builder
+        .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
+            .withLabel("Table Reference"))
+        .add(DisplayData.item("writeDisposition", writeDisposition.toString())
+            .withLabel("Write Disposition"))
+        .add(DisplayData.item("createDisposition", createDisposition.toString())
+            .withLabel("Create Disposition"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
new file mode 100644
index 0000000..29680ad
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.List;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.FileIOChannelFactory;
+import org.apache.beam.sdk.util.GcsIOChannelFactory;
+import org.apache.beam.sdk.util.GcsUtil;
+import org.apache.beam.sdk.util.GcsUtil.GcsUtilFactory;
+import org.apache.beam.sdk.util.IOChannelFactory;
+import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Writes partitions to BigQuery tables.
+ */
+class WriteTables extends DoFn<KV<Long, Iterable<List<String>>>, String> {
+  private static final Logger LOG = LoggerFactory.getLogger(WriteTables.class);
+
+  private final boolean singlePartition;
+  private final BigQueryServices bqServices;
+  private final PCollectionView<String> jobIdToken;
+  private final String tempFilePrefix;
+  private final ValueProvider<String> jsonTableRef;
+  private final ValueProvider<String> jsonSchema;
+  private final WriteDisposition writeDisposition;
+  private final CreateDisposition createDisposition;
+  @Nullable
+  private final String tableDescription;
+
+  public WriteTables(
+      boolean singlePartition,
+      BigQueryServices bqServices,
+      PCollectionView<String> jobIdToken,
+      String tempFilePrefix,
+      ValueProvider<String> jsonTableRef,
+      ValueProvider<String> jsonSchema,
+      WriteDisposition writeDisposition,
+      CreateDisposition createDisposition,
+      @Nullable String tableDescription) {
+    this.singlePartition = singlePartition;
+    this.bqServices = bqServices;
+    this.jobIdToken = jobIdToken;
+    this.tempFilePrefix = tempFilePrefix;
+    this.jsonTableRef = jsonTableRef;
+    this.jsonSchema = jsonSchema;
+    this.writeDisposition = writeDisposition;
+    this.createDisposition = createDisposition;
+    this.tableDescription = tableDescription;
+  }
+
+  @ProcessElement
+  public void processElement(ProcessContext c) throws Exception {
+    List<String> partition = Lists.newArrayList(c.element().getValue()).get(0);
+    String jobIdPrefix = String.format(
+        c.sideInput(jobIdToken) + "_%05d", c.element().getKey());
+    TableReference ref = BigQueryHelpers.fromJsonString(jsonTableRef.get(),
+        TableReference.class);
+    if (!singlePartition) {
+      ref.setTableId(jobIdPrefix);
+    }
+
+    load(
+        bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
+        bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
+        jobIdPrefix,
+        ref,
+        BigQueryHelpers.fromJsonString(
+            jsonSchema == null ? null : jsonSchema.get(), TableSchema.class),
+        partition,
+        writeDisposition,
+        createDisposition,
+        tableDescription);
+    c.output(BigQueryHelpers.toJsonString(ref));
+
+    removeTemporaryFiles(c.getPipelineOptions(), tempFilePrefix, partition);
+  }
+
+  private void load(
+      JobService jobService,
+      DatasetService datasetService,
+      String jobIdPrefix,
+      TableReference ref,
+      @Nullable TableSchema schema,
+      List<String> gcsUris,
+      WriteDisposition writeDisposition,
+      CreateDisposition createDisposition,
+      @Nullable String tableDescription) throws InterruptedException, IOException {
+    JobConfigurationLoad loadConfig = new JobConfigurationLoad()
+        .setDestinationTable(ref)
+        .setSchema(schema)
+        .setSourceUris(gcsUris)
+        .setWriteDisposition(writeDisposition.name())
+        .setCreateDisposition(createDisposition.name())
+        .setSourceFormat("NEWLINE_DELIMITED_JSON");
+
+    String projectId = ref.getProjectId();
+    Job lastFailedLoadJob = null;
+    for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
+      String jobId = jobIdPrefix + "-" + i;
+      JobReference jobRef = new JobReference()
+          .setProjectId(projectId)
+          .setJobId(jobId);
+      jobService.startLoadJob(jobRef, loadConfig);
+      Job loadJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
+      Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
+      switch (jobStatus) {
+        case SUCCEEDED:
+          if (tableDescription != null) {
+            datasetService.patchTableDescription(ref, tableDescription);
+          }
+          return;
+        case UNKNOWN:
+          throw new RuntimeException(String.format(
+              "UNKNOWN status of load job [%s]: %s.", jobId,
+              BigQueryHelpers.jobToPrettyString(loadJob)));
+        case FAILED:
+          lastFailedLoadJob = loadJob;
+          continue;
+        default:
+          throw new IllegalStateException(String.format(
+              "Unexpected status [%s] of load job: %s.",
+              jobStatus, BigQueryHelpers.jobToPrettyString(loadJob)));
+      }
+    }
+    throw new RuntimeException(String.format(
+        "Failed to create load job with id prefix %s, "
+            + "reached max retries: %d, last failed load job: %s.",
+        jobIdPrefix,
+        Write.MAX_RETRY_JOBS,
+        BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
+  }
+
+  static void removeTemporaryFiles(
+      PipelineOptions options,
+      String tempFilePrefix,
+      Collection<String> files)
+      throws IOException {
+    IOChannelFactory factory = IOChannelUtils.getFactory(tempFilePrefix);
+    if (factory instanceof GcsIOChannelFactory) {
+      GcsUtil gcsUtil = new GcsUtilFactory().create(options);
+      gcsUtil.remove(files);
+    } else if (factory instanceof FileIOChannelFactory) {
+      for (String filename : files) {
+        LOG.debug("Removing file {}", filename);
+        boolean exists = Files.deleteIfExists(Paths.get(filename));
+        if (!exists) {
+          LOG.debug("{} does not exist.", filename);
+        }
+      }
+    } else {
+      throw new IOException("Unrecognized file system.");
+    }
+  }
+
+  @Override
+  public void populateDisplayData(DisplayData.Builder builder) {
+    super.populateDisplayData(builder);
+
+    builder
+        .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
+            .withLabel("Temporary File Prefix"))
+        .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
+            .withLabel("Table Reference"))
+        .addIfNotNull(DisplayData.item("jsonSchema", jsonSchema)
+            .withLabel("Table Schema"))
+        .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
+            .withLabel("Table Description"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index eca95b9..e7db8a2 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -102,14 +102,10 @@ import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.CountingInput;
 import org.apache.beam.sdk.io.CountingSource;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.TableRowWriter;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WritePartition;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteRename;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteTables;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
 import org.apache.beam.sdk.io.gcp.bigquery.PassThroughThenCleanup.CleanupOperation;
@@ -654,7 +650,9 @@ public class BigQueryIOTest implements Serializable {
 
   @Rule public final transient TestPipeline p = TestPipeline.create();
   @Rule public transient ExpectedException thrown = ExpectedException.none();
-  @Rule public transient ExpectedLogs logged = ExpectedLogs.none(BigQueryIO.class);
+  @Rule public transient ExpectedLogs loggedBigQueryIO = ExpectedLogs.none(BigQueryIO.class);
+  @Rule public transient ExpectedLogs loggedWriteRename = ExpectedLogs.none(WriteRename.class);
+  @Rule public transient ExpectedLogs loggedWriteTables = ExpectedLogs.none(WriteTables.class);
   @Rule public transient TemporaryFolder testFolder = new TemporaryFolder();
   @Mock(extraInterfaces = Serializable.class)
   public transient BigQueryServices.JobService mockJobService;
@@ -1577,7 +1575,7 @@ public class BigQueryIOTest implements Serializable {
 
   @Test
   public void testStreamingWriteFnCreateNever() throws Exception {
-    BigQueryIO.StreamingWriteFn fn = new BigQueryIO.StreamingWriteFn(
+    StreamingWriteFn fn = new StreamingWriteFn(
         null, CreateDisposition.CREATE_NEVER, null, new FakeBigQueryServices());
     assertEquals(BigQueryHelpers.parseTableSpec("dataset.table"),
         fn.getOrCreateTable(null, "dataset.table"));
@@ -2230,9 +2228,9 @@ public class BigQueryIOTest implements Serializable {
     testNumFiles(tempDir, 0);
 
     for (String fileName : fileNames) {
-      logged.verifyDebug("Removing file " + fileName);
+      loggedWriteTables.verifyDebug("Removing file " + fileName);
     }
-    logged.verifyDebug(fileNames.get(numFiles) + " does not exist.");
+    loggedWriteTables.verifyDebug(fileNames.get(numFiles) + " does not exist.");
   }
 
   @Test
@@ -2297,11 +2295,14 @@ public class BigQueryIOTest implements Serializable {
     WriteRename.removeTemporaryTables(mockDatasetService, tableRefs);
 
     for (TableReference ref : tableRefs) {
-      logged.verifyDebug("Deleting table " + toJsonString(ref));
+      loggedWriteRename.verifyDebug("Deleting table " + toJsonString(ref));
     }
-    logged.verifyWarn("Failed to delete the table " + toJsonString(tableRefs.get(0)));
-    logged.verifyNotLogged("Failed to delete the table " + toJsonString(tableRefs.get(1)));
-    logged.verifyNotLogged("Failed to delete the table " + toJsonString(tableRefs.get(2)));
+    loggedWriteRename.verifyWarn("Failed to delete the table "
+        + toJsonString(tableRefs.get(0)));
+    loggedWriteRename.verifyNotLogged("Failed to delete the table "
+        + toJsonString(tableRefs.get(1)));
+    loggedWriteRename.verifyNotLogged("Failed to delete the table "
+        + toJsonString(tableRefs.get(2)));
   }
 
   /** Test options. **/
@@ -2367,8 +2368,8 @@ public class BigQueryIOTest implements Serializable {
   public void testTagWithUniqueIdsAndTableProjectNotNullWithNvp() {
     BigQueryOptions bqOptions = PipelineOptionsFactory.as(BigQueryOptions.class);
     bqOptions.setProject("project");
-    BigQueryIO.TagWithUniqueIdsAndTable<TableRow> tag =
-        new BigQueryIO.TagWithUniqueIdsAndTable<TableRow>(
+    TagWithUniqueIdsAndTable<TableRow> tag =
+        new TagWithUniqueIdsAndTable<TableRow>(
             bqOptions, NestedValueProvider.of(
                 StaticValueProvider.of("data_set.table_name"),
                 new TableSpecToTableRef()), null, null);
@@ -2423,12 +2424,12 @@ public class BigQueryIOTest implements Serializable {
 
   @Test
   public void testShardedKeyCoderIsSerializableWithWellKnownCoderType() {
-    CoderProperties.coderSerializable(BigQueryIO.ShardedKeyCoder.of(GlobalWindow.Coder.INSTANCE));
+    CoderProperties.coderSerializable(ShardedKeyCoder.of(GlobalWindow.Coder.INSTANCE));
   }
 
   @Test
   public void testTableRowInfoCoderSerializable() {
-    CoderProperties.coderSerializable(BigQueryIO.TableRowInfoCoder.of());
+    CoderProperties.coderSerializable(TableRowInfoCoder.of());
   }
 
   @Test
@@ -2436,8 +2437,8 @@ public class BigQueryIOTest implements Serializable {
     CoderProperties.coderSerializable(
         WindowedValue.getFullCoder(
             KvCoder.of(
-                BigQueryIO.ShardedKeyCoder.of(StringUtf8Coder.of()),
-                BigQueryIO.TableRowInfoCoder.of()),
+                ShardedKeyCoder.of(StringUtf8Coder.of()),
+                TableRowInfoCoder.of()),
             IntervalWindow.getCoder()));
   }
 }


[6/6] beam git commit: This closes #2271

Posted by jk...@apache.org.
This closes #2271


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1339dd70
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1339dd70
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1339dd70

Branch: refs/heads/master
Commit: 1339dd70623d4a1737e10d5cdc22416ffdaf6d06
Parents: 434eadb 2381dee
Author: Eugene Kirpichov <ki...@google.com>
Authored: Tue Mar 28 08:46:35 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:35 2017 -0700

----------------------------------------------------------------------
 .../sdk/io/gcp/bigquery/BatchLoadBigQuery.java  |  182 ++
 .../sdk/io/gcp/bigquery/BigQueryHelpers.java    |  339 ++++
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java    | 1915 +-----------------
 .../io/gcp/bigquery/BigQueryQuerySource.java    |  205 ++
 .../io/gcp/bigquery/BigQueryServicesImpl.java   |    2 +-
 .../sdk/io/gcp/bigquery/BigQuerySourceBase.java |  196 ++
 .../io/gcp/bigquery/BigQueryTableSource.java    |  104 +
 .../io/gcp/bigquery/PassThroughThenCleanup.java |   84 +
 .../beam/sdk/io/gcp/bigquery/ShardedKey.java    |   44 +
 .../sdk/io/gcp/bigquery/ShardedKeyCoder.java    |   87 +
 .../sdk/io/gcp/bigquery/StreamWithDeDup.java    |   98 +
 .../sdk/io/gcp/bigquery/StreamingWriteFn.java   |  186 ++
 .../beam/sdk/io/gcp/bigquery/TableRowInfo.java  |   34 +
 .../sdk/io/gcp/bigquery/TableRowInfoCoder.java  |   68 +
 .../sdk/io/gcp/bigquery/TableRowWriter.java     |   84 +
 .../gcp/bigquery/TagWithUniqueIdsAndTable.java  |  135 ++
 .../sdk/io/gcp/bigquery/TransformingSource.java |  136 ++
 .../beam/sdk/io/gcp/bigquery/WriteBundles.java  |   82 +
 .../sdk/io/gcp/bigquery/WritePartition.java     |   79 +
 .../beam/sdk/io/gcp/bigquery/WriteRename.java   |  180 ++
 .../beam/sdk/io/gcp/bigquery/WriteTables.java   |  213 ++
 .../sdk/io/gcp/bigquery/BigQueryIOTest.java     |   93 +-
 .../sdk/io/gcp/bigquery/BigQueryUtilTest.java   |   12 +-
 23 files changed, 2627 insertions(+), 1931 deletions(-)
----------------------------------------------------------------------