You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jk...@apache.org on 2017/03/28 15:51:43 UTC
[1/6] beam git commit: Move a few more functions into BigQueryHelper
Repository: beam
Updated Branches:
refs/heads/master 434eadb53 -> 1339dd706
Move a few more functions into BigQueryHelper
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2381dee4
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2381dee4
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2381dee4
Branch: refs/heads/master
Commit: 2381dee48faf52882b873597d49b9490967f6adb
Parents: d6ef010
Author: Reuven Lax <re...@google.com>
Authored: Sat Mar 18 13:03:25 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:15 2017 -0700
----------------------------------------------------------------------
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 40 +++++++++++-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 67 ++------------------
2 files changed, 44 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2381dee4/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 37ff124..c5156e9 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -24,6 +24,7 @@ import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobStatus;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableSchema;
+import com.google.cloud.hadoop.util.ApiErrorExtractor;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
@@ -196,6 +197,43 @@ class BigQueryHelpers {
}
}
+ static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
+ try {
+ datasetService.getDataset(table.getProjectId(), table.getDatasetId());
+ } catch (Exception e) {
+ ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
+ if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
+ throw new IllegalArgumentException(
+ String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", toTableSpec(table)), e);
+ } else if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ } else {
+ throw new RuntimeException(
+ String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "dataset",
+ toTableSpec(table)),
+ e);
+ }
+ }
+ }
+
+ static void verifyTablePresence(DatasetService datasetService, TableReference table) {
+ try {
+ datasetService.getTable(table);
+ } catch (Exception e) {
+ ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
+ if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
+ throw new IllegalArgumentException(
+ String.format(RESOURCE_NOT_FOUND_ERROR, "table", toTableSpec(table)), e);
+ } else if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ } else {
+ throw new RuntimeException(
+ String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "table",
+ toTableSpec(table)), e);
+ }
+ }
+ }
+
@VisibleForTesting
static class JsonSchemaToTableSchema
implements SerializableFunction<String, TableSchema> {
@@ -283,7 +321,7 @@ class BigQueryHelpers {
implements SerializableFunction<String, TableReference> {
private final String executingProject;
- public CreateJsonTableRefFromUuid(String executingProject) {
+ CreateJsonTableRefFromUuid(String executingProject) {
this.executingProject = executingProject;
}
http://git-wip-us.apache.org/repos/asf/beam/blob/2381dee4/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 4917083..cc6ec09 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -29,22 +29,17 @@ import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.auto.value.AutoValue;
-import com.google.cloud.hadoop.util.ApiErrorExtractor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
-import com.google.common.io.CountingOutputStream;
+
import java.io.IOException;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
-import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.List;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
import org.apache.beam.sdk.coders.Coder;
-import org.apache.beam.sdk.coders.Coder.Context;
import org.apache.beam.sdk.coders.TableRowJsonCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.BoundedSource;
@@ -70,7 +65,6 @@ import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.util.GcsUtil.GcsUtilFactory;
import org.apache.beam.sdk.util.IOChannelFactory;
import org.apache.beam.sdk.util.IOChannelUtils;
-import org.apache.beam.sdk.util.MimeTypes;
import org.apache.beam.sdk.util.Transport;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
import org.apache.beam.sdk.values.PBegin;
@@ -229,17 +223,6 @@ public class BigQueryIO {
static final Pattern TABLE_SPEC = Pattern.compile(DATASET_TABLE_REGEXP);
- private static final String RESOURCE_NOT_FOUND_ERROR =
- "BigQuery %1$s not found for table \"%2$s\" . Please create the %1$s before pipeline"
- + " execution. If the %1$s is created by an earlier stage of the pipeline, this"
- + " validation can be disabled using #withoutValidation.";
-
- private static final String UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR =
- "Unable to confirm BigQuery %1$s presence for table \"%2$s\". If the %1$s is created by"
- + " an earlier stage of the pipeline, this validation can be disabled using"
- + " #withoutValidation.";
-
-
/**
* A formatting function that maps a TableRow to itself. This allows sending a
* {@code PCollection<TableRow>} directly to BigQueryIO.Write.
@@ -451,8 +434,8 @@ public class BigQueryIO {
checkState(table.isAccessible(), "Cannot call validate if table is dynamically set.");
// Check for source table presence for early failure notification.
DatasetService datasetService = getBigQueryServices().getDatasetService(bqOptions);
- verifyDatasetPresence(datasetService, table.get());
- verifyTablePresence(datasetService, table.get());
+ BigQueryHelpers.verifyDatasetPresence(datasetService, table.get());
+ BigQueryHelpers.verifyTablePresence(datasetService, table.get());
} else if (getValidate() && getQuery() != null) {
checkState(getQuery().isAccessible(), "Cannot call validate if query is dynamically set.");
JobService jobService = getBigQueryServices().getJobService(bqOptions);
@@ -953,9 +936,9 @@ public class BigQueryIO {
// Note that a presence check can fail when the table or dataset is created by an earlier
// stage of the pipeline. For these cases the #withoutValidation method can be used to
// disable the check.
- verifyDatasetPresence(datasetService, table);
+ BigQueryHelpers.verifyDatasetPresence(datasetService, table);
if (getCreateDisposition() == BigQueryIO.Write.CreateDisposition.CREATE_NEVER) {
- verifyTablePresence(datasetService, table);
+ BigQueryHelpers.verifyTablePresence(datasetService, table);
}
if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table);
@@ -1085,46 +1068,6 @@ public class BigQueryIO {
}
- private static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
- try {
- datasetService.getDataset(table.getProjectId(), table.getDatasetId());
- } catch (Exception e) {
- ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
- if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
- throw new IllegalArgumentException(
- String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", BigQueryHelpers.toTableSpec(table)),
- e);
- } else if (e instanceof RuntimeException) {
- throw (RuntimeException) e;
- } else {
- throw new RuntimeException(
- String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "dataset",
- BigQueryHelpers.toTableSpec(table)),
- e);
- }
- }
- }
-
- private static void verifyTablePresence(DatasetService datasetService, TableReference table) {
- try {
- datasetService.getTable(table);
- } catch (Exception e) {
- ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
- if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
- throw new IllegalArgumentException(
- String.format(RESOURCE_NOT_FOUND_ERROR, "table", BigQueryHelpers.toTableSpec(table)),
- e);
- } else if (e instanceof RuntimeException) {
- throw (RuntimeException) e;
- } else {
- throw new RuntimeException(
- String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "table",
- BigQueryHelpers.toTableSpec(table)),
- e);
- }
- }
- }
-
/**
* Clear the cached map of created tables. Used for testing.
*/
[5/6] beam git commit: Refactor BigQueryIO helper methods into a
BigQueryHelpers class. Move helper transforms for BigQueryIO.Read into
individual files. Change private to package in BigQueryHelpers.
Posted by jk...@apache.org.
Refactor BigQueryIO helper methods into a BigQueryHelpers class.
Move helper transforms for BigQueryIO.Read into individual files.
Change private to package in BigQueryHelpers.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2cc2e81f
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2cc2e81f
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2cc2e81f
Branch: refs/heads/master
Commit: 2cc2e81f241e1d5d590f12d4a8491273908e2cce
Parents: 434eadb
Author: Reuven Lax <re...@google.com>
Authored: Fri Mar 17 15:15:16 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:15 2017 -0700
----------------------------------------------------------------------
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 243 ++++++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 853 ++-----------------
.../io/gcp/bigquery/BigQueryQuerySource.java | 186 ++++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 2 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 178 ++++
.../io/gcp/bigquery/BigQueryTableSource.java | 86 ++
.../io/gcp/bigquery/PassThroughThenCleanup.java | 66 ++
.../sdk/io/gcp/bigquery/TransformingSource.java | 118 +++
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 54 +-
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 12 +-
10 files changed, 970 insertions(+), 828 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
new file mode 100644
index 0000000..9fba938
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -0,0 +1,243 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobStatus;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+
+
+/**
+ * A set of helper functions and classes used by {@link BigQueryIO}.
+ */
+public class BigQueryHelpers {
+ @Nullable
+ /**
+ * Return a displayable string representation for a {@link TableReference}.
+ */
+ static ValueProvider<String> displayTable(
+ @Nullable ValueProvider<TableReference> table) {
+ if (table == null) {
+ return null;
+ }
+ return NestedValueProvider.of(table, new TableRefToTableSpec());
+ }
+
+ /**
+ * Returns a canonical string representation of the {@link TableReference}.
+ */
+ static String toTableSpec(TableReference ref) {
+ StringBuilder sb = new StringBuilder();
+ if (ref.getProjectId() != null) {
+ sb.append(ref.getProjectId());
+ sb.append(":");
+ }
+
+ sb.append(ref.getDatasetId()).append('.').append(ref.getTableId());
+ return sb.toString();
+ }
+
+ static <K, V> List<V> getOrCreateMapListValue(Map<K, List<V>> map, K key) {
+ List<V> value = map.get(key);
+ if (value == null) {
+ value = new ArrayList<>();
+ map.put(key, value);
+ }
+ return value;
+ }
+
+ /**
+ * Parse a table specification in the form
+ * {@code "[project_id]:[dataset_id].[table_id]"} or {@code "[dataset_id].[table_id]"}.
+ *
+ * <p>If the project id is omitted, the default project id is used.
+ */
+ static TableReference parseTableSpec(String tableSpec) {
+ Matcher match = BigQueryIO.TABLE_SPEC.matcher(tableSpec);
+ if (!match.matches()) {
+ throw new IllegalArgumentException(
+ "Table reference is not in [project_id]:[dataset_id].[table_id] "
+ + "format: " + tableSpec);
+ }
+
+ TableReference ref = new TableReference();
+ ref.setProjectId(match.group("PROJECT"));
+
+ return ref.setDatasetId(match.group("DATASET")).setTableId(match.group("TABLE"));
+ }
+
+ static String jobToPrettyString(@Nullable Job job) throws IOException {
+ return job == null ? "null" : job.toPrettyString();
+ }
+
+ static String statusToPrettyString(@Nullable JobStatus status) throws IOException {
+ return status == null ? "Unknown status: null." : status.toPrettyString();
+ }
+
+ static Status parseStatus(@Nullable Job job) {
+ if (job == null) {
+ return Status.UNKNOWN;
+ }
+ JobStatus status = job.getStatus();
+ if (status.getErrorResult() != null) {
+ return Status.FAILED;
+ } else if (status.getErrors() != null && !status.getErrors().isEmpty()) {
+ return Status.FAILED;
+ } else {
+ return Status.SUCCEEDED;
+ }
+ }
+
+ @VisibleForTesting
+ static String toJsonString(Object item) {
+ if (item == null) {
+ return null;
+ }
+ try {
+ return BigQueryIO.JSON_FACTORY.toString(item);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format("Cannot serialize %s to a JSON string.", item.getClass().getSimpleName()),
+ e);
+ }
+ }
+
+ @VisibleForTesting
+ static <T> T fromJsonString(String json, Class<T> clazz) {
+ if (json == null) {
+ return null;
+ }
+ try {
+ return BigQueryIO.JSON_FACTORY.fromString(json, clazz);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format("Cannot deserialize %s from a JSON string: %s.", clazz, json),
+ e);
+ }
+ }
+
+ /**
+ * Returns a randomUUID string.
+ *
+ * <p>{@code '-'} is removed because BigQuery doesn't allow it in dataset id.
+ */
+ static String randomUUIDString() {
+ return UUID.randomUUID().toString().replaceAll("-", "");
+ }
+
+ @VisibleForTesting
+ static class JsonSchemaToTableSchema
+ implements SerializableFunction<String, TableSchema> {
+ @Override
+ public TableSchema apply(String from) {
+ return fromJsonString(from, TableSchema.class);
+ }
+ }
+
+ @VisibleForTesting
+ static class BeamJobUuidToBigQueryJobUuid
+ implements SerializableFunction<String, String> {
+ @Override
+ public String apply(String from) {
+ return "beam_job_" + from;
+ }
+ }
+
+ static class TableSchemaToJsonSchema
+ implements SerializableFunction<TableSchema, String> {
+ @Override
+ public String apply(TableSchema from) {
+ return toJsonString(from);
+ }
+ }
+
+ static class JsonTableRefToTableRef
+ implements SerializableFunction<String, TableReference> {
+ @Override
+ public TableReference apply(String from) {
+ return fromJsonString(from, TableReference.class);
+ }
+ }
+
+ static class TableRefToTableSpec
+ implements SerializableFunction<TableReference, String> {
+ @Override
+ public String apply(TableReference from) {
+ return toTableSpec(from);
+ }
+ }
+
+ static class TableRefToJson
+ implements SerializableFunction<TableReference, String> {
+ @Override
+ public String apply(TableReference from) {
+ return toJsonString(from);
+ }
+ }
+
+ static class TableRefToProjectId
+ implements SerializableFunction<TableReference, String> {
+ @Override
+ public String apply(TableReference from) {
+ return from.getProjectId();
+ }
+ }
+
+ @VisibleForTesting
+ static class TableSpecToTableRef
+ implements SerializableFunction<String, TableReference> {
+ @Override
+ public TableReference apply(String from) {
+ return parseTableSpec(from);
+ }
+ }
+
+ @VisibleForTesting
+ static class CreatePerBeamJobUuid
+ implements SerializableFunction<String, String> {
+ private final String stepUuid;
+
+ CreatePerBeamJobUuid(String stepUuid) {
+ this.stepUuid = stepUuid;
+ }
+
+ @Override
+ public String apply(String jobUuid) {
+ return stepUuid + "_" + jobUuid.replaceAll("-", "");
+ }
+ }
+
+ @VisibleForTesting
+ static class CreateJsonTableRefFromUuid
+ implements SerializableFunction<String, TableReference> {
+ private final String executingProject;
+
+ public CreateJsonTableRefFromUuid(String executingProject) {
+ this.executingProject = executingProject;
+ }
+
+ @Override
+ public TableReference apply(String jobUuid) {
+ String queryTempDatasetId = "temp_dataset_" + jobUuid;
+ String queryTempTableId = "temp_table_" + jobUuid;
+ TableReference queryTempTableRef = new TableReference()
+ .setProjectId(executingProject)
+ .setDatasetId(queryTempDatasetId)
+ .setTableId(queryTempTableId);
+ return queryTempTableRef;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index d195afd..3f2f3e8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -25,13 +25,11 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.api.client.json.JsonFactory;
import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobConfigurationExtract;
import com.google.api.services.bigquery.model.JobConfigurationLoad;
import com.google.api.services.bigquery.model.JobConfigurationQuery;
import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.JobStatistics;
-import com.google.api.services.bigquery.model.JobStatus;
import com.google.api.services.bigquery.model.Table;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
@@ -39,38 +37,31 @@ import com.google.api.services.bigquery.model.TableSchema;
import com.google.auto.value.AutoValue;
import com.google.cloud.hadoop.util.ApiErrorExtractor;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.io.CountingOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.ObjectInputStream;
import java.io.OutputStream;
-import java.io.Serializable;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.NoSuchElementException;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
-import org.apache.avro.generic.GenericRecord;
+
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.Coder;
@@ -82,8 +73,16 @@ import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.TableRowJsonCoder;
import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
-import org.apache.beam.sdk.io.AvroSource;
import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.BeamJobUuidToBigQueryJobUuid;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreateJsonTableRefFromUuid;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreatePerBeamJobUuid;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonTableRefToTableRef;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToTableSpec;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.metrics.Counter;
@@ -131,7 +130,6 @@ import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.TupleTagList;
import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.ValueInSingleWindow;
-import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -153,7 +151,7 @@ import org.slf4j.LoggerFactory;
* from the <a href="https://cloud.google.com/bigquery/client-libraries">
* BigQuery Java Client API</a>.
* Tables can be referred to as Strings, with or without the {@code projectId}.
- * A helper function is provided ({@link BigQueryIO#parseTableSpec(String)})
+ * A helper function is provided ({@link BigQueryHelpers#parseTableSpec(String)})
* that parses the following string forms into a {@link TableReference}:
*
* <ul>
@@ -253,7 +251,7 @@ public class BigQueryIO {
* Singleton instance of the JSON factory used to read and write JSON
* formatted rows.
*/
- private static final JsonFactory JSON_FACTORY = Transport.getJsonFactory();
+ static final JsonFactory JSON_FACTORY = Transport.getJsonFactory();
/**
* Project IDs must contain 6-63 lowercase letters, digits, or dashes.
@@ -281,7 +279,7 @@ public class BigQueryIO {
String.format("((?<PROJECT>%s):)?(?<DATASET>%s)\\.(?<TABLE>%s)", PROJECT_ID_REGEXP,
DATASET_REGEXP, TABLE_REGEXP);
- private static final Pattern TABLE_SPEC = Pattern.compile(DATASET_TABLE_REGEXP);
+ static final Pattern TABLE_SPEC = Pattern.compile(DATASET_TABLE_REGEXP);
private static final String RESOURCE_NOT_FOUND_ERROR =
"BigQuery %1$s not found for table \"%2$s\" . Please create the %1$s before pipeline"
@@ -293,152 +291,6 @@ public class BigQueryIO {
+ " an earlier stage of the pipeline, this validation can be disabled using"
+ " #withoutValidation.";
- /**
- * Parse a table specification in the form
- * {@code "[project_id]:[dataset_id].[table_id]"} or {@code "[dataset_id].[table_id]"}.
- *
- * <p>If the project id is omitted, the default project id is used.
- */
- public static TableReference parseTableSpec(String tableSpec) {
- Matcher match = TABLE_SPEC.matcher(tableSpec);
- if (!match.matches()) {
- throw new IllegalArgumentException(
- "Table reference is not in [project_id]:[dataset_id].[table_id] "
- + "format: " + tableSpec);
- }
-
- TableReference ref = new TableReference();
- ref.setProjectId(match.group("PROJECT"));
-
- return ref.setDatasetId(match.group("DATASET")).setTableId(match.group("TABLE"));
- }
-
- /**
- * Returns a canonical string representation of the {@link TableReference}.
- */
- public static String toTableSpec(TableReference ref) {
- StringBuilder sb = new StringBuilder();
- if (ref.getProjectId() != null) {
- sb.append(ref.getProjectId());
- sb.append(":");
- }
-
- sb.append(ref.getDatasetId()).append('.').append(ref.getTableId());
- return sb.toString();
- }
-
- @VisibleForTesting
- static class JsonSchemaToTableSchema
- implements SerializableFunction<String, TableSchema> {
- @Override
- public TableSchema apply(String from) {
- return fromJsonString(from, TableSchema.class);
- }
- }
-
- private static class TableSchemaToJsonSchema
- implements SerializableFunction<TableSchema, String> {
- @Override
- public String apply(TableSchema from) {
- return toJsonString(from);
- }
- }
-
- private static class JsonTableRefToTableRef
- implements SerializableFunction<String, TableReference> {
- @Override
- public TableReference apply(String from) {
- return fromJsonString(from, TableReference.class);
- }
- }
-
- private static class TableRefToTableSpec
- implements SerializableFunction<TableReference, String> {
- @Override
- public String apply(TableReference from) {
- return toTableSpec(from);
- }
- }
-
- private static class TableRefToJson
- implements SerializableFunction<TableReference, String> {
- @Override
- public String apply(TableReference from) {
- return toJsonString(from);
- }
- }
-
- private static class TableRefToProjectId
- implements SerializableFunction<TableReference, String> {
- @Override
- public String apply(TableReference from) {
- return from.getProjectId();
- }
- }
-
- @VisibleForTesting
- static class TableSpecToTableRef
- implements SerializableFunction<String, TableReference> {
- @Override
- public TableReference apply(String from) {
- return parseTableSpec(from);
- }
- }
-
- @VisibleForTesting
- static class BeamJobUuidToBigQueryJobUuid
- implements SerializableFunction<String, String> {
- @Override
- public String apply(String from) {
- return "beam_job_" + from;
- }
- }
-
- @VisibleForTesting
- static class CreatePerBeamJobUuid
- implements SerializableFunction<String, String> {
- private final String stepUuid;
-
- private CreatePerBeamJobUuid(String stepUuid) {
- this.stepUuid = stepUuid;
- }
-
- @Override
- public String apply(String jobUuid) {
- return stepUuid + "_" + jobUuid.replaceAll("-", "");
- }
- }
-
- @VisibleForTesting
- static class CreateJsonTableRefFromUuid
- implements SerializableFunction<String, TableReference> {
- private final String executingProject;
-
- private CreateJsonTableRefFromUuid(String executingProject) {
- this.executingProject = executingProject;
- }
-
- @Override
- public TableReference apply(String jobUuid) {
- String queryTempDatasetId = "temp_dataset_" + jobUuid;
- String queryTempTableId = "temp_table_" + jobUuid;
- TableReference queryTempTableRef = new TableReference()
- .setProjectId(executingProject)
- .setDatasetId(queryTempDatasetId)
- .setTableId(queryTempTableId);
- return queryTempTableRef;
- }
- }
-
- @Nullable
- private static ValueProvider<String> displayTable(
- @Nullable ValueProvider<TableReference> table) {
- if (table == null) {
- return null;
- }
- return NestedValueProvider.of(table, new TableRefToTableSpec());
- }
-
/**
* A formatting function that maps a TableRow to itself. This allows sending a
@@ -556,7 +408,7 @@ public class BigQueryIO {
* Read from table specified by a {@link TableReference}.
*/
public Read from(TableReference table) {
- return from(StaticValueProvider.of(toTableSpec(table)));
+ return from(StaticValueProvider.of(BigQueryHelpers.toTableSpec(table)));
}
private static final String QUERY_VALIDATION_FAILURE_ERROR =
@@ -672,7 +524,7 @@ public class BigQueryIO {
@Override
public PCollection<TableRow> expand(PBegin input) {
- String stepUuid = randomUUIDString();
+ String stepUuid = BigQueryHelpers.randomUUIDString();
BigQueryOptions bqOptions = input.getPipeline().getOptions().as(BigQueryOptions.class);
ValueProvider<String> jobUuid = NestedValueProvider.of(
StaticValueProvider.of(bqOptions.getJobName()), new CreatePerBeamJobUuid(stepUuid));
@@ -752,7 +604,7 @@ public class BigQueryIO {
public void populateDisplayData(DisplayData.Builder builder) {
super.populateDisplayData(builder);
builder
- .addIfNotNull(DisplayData.item("table", displayTable(getTableProvider()))
+ .addIfNotNull(DisplayData.item("table", BigQueryHelpers.displayTable(getTableProvider()))
.withLabel("Table"))
.addIfNotNull(DisplayData.item("query", getQuery())
.withLabel("Query"))
@@ -787,7 +639,7 @@ public class BigQueryIO {
TableReference tableRef = table.get();
tableRef.setProjectId(bqOptions.getProject());
return NestedValueProvider.of(StaticValueProvider.of(
- toJsonString(tableRef)), new JsonTableRefToTableRef());
+ BigQueryHelpers.toJsonString(tableRef)), new JsonTableRefToTableRef());
}
return table;
}
@@ -810,541 +662,15 @@ public class BigQueryIO {
}
}
- /**
- * A {@link PTransform} that invokes {@link CleanupOperation} after the input {@link PCollection}
- * has been processed.
- */
- @VisibleForTesting
- static class PassThroughThenCleanup<T> extends PTransform<PCollection<T>, PCollection<T>> {
-
- private CleanupOperation cleanupOperation;
-
- PassThroughThenCleanup(CleanupOperation cleanupOperation) {
- this.cleanupOperation = cleanupOperation;
- }
-
- @Override
- public PCollection<T> expand(PCollection<T> input) {
- TupleTag<T> mainOutput = new TupleTag<>();
- TupleTag<Void> cleanupSignal = new TupleTag<>();
- PCollectionTuple outputs = input.apply(ParDo.of(new IdentityFn<T>())
- .withOutputTags(mainOutput, TupleTagList.of(cleanupSignal)));
-
- PCollectionView<Void> cleanupSignalView = outputs.get(cleanupSignal)
- .setCoder(VoidCoder.of())
- .apply(View.<Void>asSingleton().withDefaultValue(null));
-
- input.getPipeline()
- .apply("Create(CleanupOperation)", Create.of(cleanupOperation))
- .apply("Cleanup", ParDo.of(
- new DoFn<CleanupOperation, Void>() {
- @ProcessElement
- public void processElement(ProcessContext c)
- throws Exception {
- c.element().cleanup(c.getPipelineOptions());
- }
- }).withSideInputs(cleanupSignalView));
-
- return outputs.get(mainOutput);
- }
-
- private static class IdentityFn<T> extends DoFn<T, T> {
- @ProcessElement
- public void processElement(ProcessContext c) {
- c.output(c.element());
- }
- }
-
- abstract static class CleanupOperation implements Serializable {
- abstract void cleanup(PipelineOptions options) throws Exception;
- }
- }
-
- /**
- * A {@link BigQuerySourceBase} for reading BigQuery tables.
- */
- @VisibleForTesting
- static class BigQueryTableSource extends BigQuerySourceBase {
-
- static BigQueryTableSource create(
- ValueProvider<String> jobIdToken,
- ValueProvider<TableReference> table,
- String extractDestinationDir,
- BigQueryServices bqServices,
- ValueProvider<String> executingProject) {
- return new BigQueryTableSource(
- jobIdToken, table, extractDestinationDir, bqServices, executingProject);
- }
-
- private final ValueProvider<String> jsonTable;
- private final AtomicReference<Long> tableSizeBytes;
-
- private BigQueryTableSource(
- ValueProvider<String> jobIdToken,
- ValueProvider<TableReference> table,
- String extractDestinationDir,
- BigQueryServices bqServices,
- ValueProvider<String> executingProject) {
- super(jobIdToken, extractDestinationDir, bqServices, executingProject);
- this.jsonTable = NestedValueProvider.of(checkNotNull(table, "table"), new TableRefToJson());
- this.tableSizeBytes = new AtomicReference<>();
- }
-
- @Override
- protected TableReference getTableToExtract(BigQueryOptions bqOptions) throws IOException {
- checkState(jsonTable.isAccessible());
- return JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
- }
-
- @Override
- public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
- BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
- checkState(jsonTable.isAccessible());
- TableReference tableRef = JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
- return new BigQueryReader(this, bqServices.getReaderFromTable(bqOptions, tableRef));
- }
-
- @Override
- public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
- if (tableSizeBytes.get() == null) {
- TableReference table = JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
-
- Long numBytes = bqServices.getDatasetService(options.as(BigQueryOptions.class))
- .getTable(table).getNumBytes();
- tableSizeBytes.compareAndSet(null, numBytes);
- }
- return tableSizeBytes.get();
- }
-
- @Override
- protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
- // Do nothing.
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
- builder.add(DisplayData.item("table", jsonTable));
- }
- }
-
- /**
- * A {@link BigQuerySourceBase} for querying BigQuery tables.
- */
- @VisibleForTesting
- static class BigQueryQuerySource extends BigQuerySourceBase {
-
- static BigQueryQuerySource create(
- ValueProvider<String> jobIdToken,
- ValueProvider<String> query,
- ValueProvider<TableReference> queryTempTableRef,
- Boolean flattenResults,
- Boolean useLegacySql,
- String extractDestinationDir,
- BigQueryServices bqServices) {
- return new BigQueryQuerySource(
- jobIdToken,
- query,
- queryTempTableRef,
- flattenResults,
- useLegacySql,
- extractDestinationDir,
- bqServices);
- }
-
- private final ValueProvider<String> query;
- private final ValueProvider<String> jsonQueryTempTable;
- private final Boolean flattenResults;
- private final Boolean useLegacySql;
- private transient AtomicReference<JobStatistics> dryRunJobStats;
-
- private BigQueryQuerySource(
- ValueProvider<String> jobIdToken,
- ValueProvider<String> query,
- ValueProvider<TableReference> queryTempTableRef,
- Boolean flattenResults,
- Boolean useLegacySql,
- String extractDestinationDir,
- BigQueryServices bqServices) {
- super(jobIdToken, extractDestinationDir, bqServices,
- NestedValueProvider.of(
- checkNotNull(queryTempTableRef, "queryTempTableRef"), new TableRefToProjectId()));
- this.query = checkNotNull(query, "query");
- this.jsonQueryTempTable = NestedValueProvider.of(
- queryTempTableRef, new TableRefToJson());
- this.flattenResults = checkNotNull(flattenResults, "flattenResults");
- this.useLegacySql = checkNotNull(useLegacySql, "useLegacySql");
- this.dryRunJobStats = new AtomicReference<>();
- }
-
- @Override
- public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
- BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
- return dryRunQueryIfNeeded(bqOptions).getTotalBytesProcessed();
- }
-
- @Override
- public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
- BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
- return new BigQueryReader(this, bqServices.getReaderFromQuery(
- bqOptions, executingProject.get(), createBasicQueryConfig()));
- }
-
- @Override
- protected TableReference getTableToExtract(BigQueryOptions bqOptions)
- throws IOException, InterruptedException {
- // 1. Find the location of the query.
- String location = null;
- List<TableReference> referencedTables =
- dryRunQueryIfNeeded(bqOptions).getQuery().getReferencedTables();
- DatasetService tableService = bqServices.getDatasetService(bqOptions);
- if (referencedTables != null && !referencedTables.isEmpty()) {
- TableReference queryTable = referencedTables.get(0);
- location = tableService.getTable(queryTable).getLocation();
- }
-
- // 2. Create the temporary dataset in the query location.
- TableReference tableToExtract =
- JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
- tableService.createDataset(
- tableToExtract.getProjectId(),
- tableToExtract.getDatasetId(),
- location,
- "Dataset for BigQuery query job temporary table");
-
- // 3. Execute the query.
- String queryJobId = jobIdToken.get() + "-query";
- executeQuery(
- executingProject.get(),
- queryJobId,
- tableToExtract,
- bqServices.getJobService(bqOptions));
- return tableToExtract;
- }
-
- @Override
- protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
- checkState(jsonQueryTempTable.isAccessible());
- TableReference tableToRemove =
- JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
-
- DatasetService tableService = bqServices.getDatasetService(bqOptions);
- tableService.deleteTable(tableToRemove);
- tableService.deleteDataset(tableToRemove.getProjectId(), tableToRemove.getDatasetId());
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
- builder.add(DisplayData.item("query", query));
- }
-
- private synchronized JobStatistics dryRunQueryIfNeeded(BigQueryOptions bqOptions)
- throws InterruptedException, IOException {
- if (dryRunJobStats.get() == null) {
- JobStatistics jobStats = bqServices.getJobService(bqOptions).dryRunQuery(
- executingProject.get(), createBasicQueryConfig());
- dryRunJobStats.compareAndSet(null, jobStats);
- }
- return dryRunJobStats.get();
- }
-
- private void executeQuery(
- String executingProject,
- String jobId,
- TableReference destinationTable,
- JobService jobService) throws IOException, InterruptedException {
- JobReference jobRef = new JobReference()
- .setProjectId(executingProject)
- .setJobId(jobId);
-
- JobConfigurationQuery queryConfig = createBasicQueryConfig()
- .setAllowLargeResults(true)
- .setCreateDisposition("CREATE_IF_NEEDED")
- .setDestinationTable(destinationTable)
- .setPriority("BATCH")
- .setWriteDisposition("WRITE_EMPTY");
-
- jobService.startQueryJob(jobRef, queryConfig);
- Job job = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
- if (parseStatus(job) != Status.SUCCEEDED) {
- throw new IOException(String.format(
- "Query job %s failed, status: %s.", jobId, statusToPrettyString(job.getStatus())));
- }
- }
-
- private JobConfigurationQuery createBasicQueryConfig() {
- return new JobConfigurationQuery()
- .setFlattenResults(flattenResults)
- .setQuery(query.get())
- .setUseLegacySql(useLegacySql);
- }
-
- private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException {
- in.defaultReadObject();
- dryRunJobStats = new AtomicReference<>();
- }
- }
-
- /**
- * An abstract {@link BoundedSource} to read a table from BigQuery.
- *
- * <p>This source uses a BigQuery export job to take a snapshot of the table on GCS, and then
- * reads in parallel from each produced file. It is implemented by {@link BigQueryTableSource},
- * and {@link BigQueryQuerySource}, depending on the configuration of the read.
- * Specifically,
- * <ul>
- * <li>{@link BigQueryTableSource} is for reading BigQuery tables</li>
- * <li>{@link BigQueryQuerySource} is for querying BigQuery tables</li>
- * </ul>
- * ...
- */
- private abstract static class BigQuerySourceBase extends BoundedSource<TableRow> {
- // The maximum number of retries to poll a BigQuery job.
- protected static final int JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
-
- protected final ValueProvider<String> jobIdToken;
- protected final String extractDestinationDir;
- protected final BigQueryServices bqServices;
- protected final ValueProvider<String> executingProject;
-
- private BigQuerySourceBase(
- ValueProvider<String> jobIdToken,
- String extractDestinationDir,
- BigQueryServices bqServices,
- ValueProvider<String> executingProject) {
- this.jobIdToken = checkNotNull(jobIdToken, "jobIdToken");
- this.extractDestinationDir = checkNotNull(extractDestinationDir, "extractDestinationDir");
- this.bqServices = checkNotNull(bqServices, "bqServices");
- this.executingProject = checkNotNull(executingProject, "executingProject");
- }
-
- @Override
- public List<BoundedSource<TableRow>> splitIntoBundles(
- long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
- BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
- TableReference tableToExtract = getTableToExtract(bqOptions);
- JobService jobService = bqServices.getJobService(bqOptions);
- String extractJobId = getExtractJobId(jobIdToken);
- List<String> tempFiles = executeExtract(extractJobId, tableToExtract, jobService);
-
- TableSchema tableSchema = bqServices.getDatasetService(bqOptions)
- .getTable(tableToExtract).getSchema();
-
- cleanupTempResource(bqOptions);
- return createSources(tempFiles, tableSchema);
- }
-
- protected abstract TableReference getTableToExtract(BigQueryOptions bqOptions) throws Exception;
-
- protected abstract void cleanupTempResource(BigQueryOptions bqOptions) throws Exception;
-
- @Override
- public void validate() {
- // Do nothing, validation is done in BigQuery.Read.
- }
-
- @Override
- public Coder<TableRow> getDefaultOutputCoder() {
- return TableRowJsonCoder.of();
- }
-
- private List<String> executeExtract(
- String jobId, TableReference table, JobService jobService)
- throws InterruptedException, IOException {
- JobReference jobRef = new JobReference()
- .setProjectId(executingProject.get())
- .setJobId(jobId);
-
- String destinationUri = getExtractDestinationUri(extractDestinationDir);
- JobConfigurationExtract extract = new JobConfigurationExtract()
- .setSourceTable(table)
- .setDestinationFormat("AVRO")
- .setDestinationUris(ImmutableList.of(destinationUri));
-
- LOG.info("Starting BigQuery extract job: {}", jobId);
- jobService.startExtractJob(jobRef, extract);
- Job extractJob =
- jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
- if (parseStatus(extractJob) != Status.SUCCEEDED) {
- throw new IOException(String.format(
- "Extract job %s failed, status: %s.",
- extractJob.getJobReference().getJobId(), statusToPrettyString(extractJob.getStatus())));
- }
-
- List<String> tempFiles = getExtractFilePaths(extractDestinationDir, extractJob);
- return ImmutableList.copyOf(tempFiles);
- }
-
- private List<BoundedSource<TableRow>> createSources(
- List<String> files, TableSchema tableSchema) throws IOException, InterruptedException {
- final String jsonSchema = JSON_FACTORY.toString(tableSchema);
-
- SerializableFunction<GenericRecord, TableRow> function =
- new SerializableFunction<GenericRecord, TableRow>() {
- @Override
- public TableRow apply(GenericRecord input) {
- return BigQueryAvroUtils.convertGenericRecordToTableRow(
- input, fromJsonString(jsonSchema, TableSchema.class));
- }};
-
- List<BoundedSource<TableRow>> avroSources = Lists.newArrayList();
- for (String fileName : files) {
- avroSources.add(new TransformingSource<>(
- AvroSource.from(fileName), function, getDefaultOutputCoder()));
- }
- return ImmutableList.copyOf(avroSources);
- }
-
- protected static class BigQueryReader extends BoundedSource.BoundedReader<TableRow> {
- private final BigQuerySourceBase source;
- private final BigQueryServices.BigQueryJsonReader reader;
-
- private BigQueryReader(
- BigQuerySourceBase source, BigQueryServices.BigQueryJsonReader reader) {
- this.source = source;
- this.reader = reader;
- }
-
- @Override
- public BoundedSource<TableRow> getCurrentSource() {
- return source;
- }
-
- @Override
- public boolean start() throws IOException {
- return reader.start();
- }
-
- @Override
- public boolean advance() throws IOException {
- return reader.advance();
- }
-
- @Override
- public TableRow getCurrent() throws NoSuchElementException {
- return reader.getCurrent();
- }
-
- @Override
- public void close() throws IOException {
- reader.close();
- }
- }
- }
-
- /**
- * A {@link BoundedSource} that reads from {@code BoundedSource<T>}
- * and transforms elements to type {@code V}.
- */
- @VisibleForTesting
- static class TransformingSource<T, V> extends BoundedSource<V> {
- private final BoundedSource<T> boundedSource;
- private final SerializableFunction<T, V> function;
- private final Coder<V> outputCoder;
-
- TransformingSource(
- BoundedSource<T> boundedSource,
- SerializableFunction<T, V> function,
- Coder<V> outputCoder) {
- this.boundedSource = checkNotNull(boundedSource, "boundedSource");
- this.function = checkNotNull(function, "function");
- this.outputCoder = checkNotNull(outputCoder, "outputCoder");
- }
-
- @Override
- public List<? extends BoundedSource<V>> splitIntoBundles(
- long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
- return Lists.transform(
- boundedSource.splitIntoBundles(desiredBundleSizeBytes, options),
- new Function<BoundedSource<T>, BoundedSource<V>>() {
- @Override
- public BoundedSource<V> apply(BoundedSource<T> input) {
- return new TransformingSource<>(input, function, outputCoder);
- }
- });
- }
-
- @Override
- public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
- return boundedSource.getEstimatedSizeBytes(options);
- }
-
- @Override
- public BoundedReader<V> createReader(PipelineOptions options) throws IOException {
- return new TransformingReader(boundedSource.createReader(options));
- }
-
- @Override
- public void validate() {
- boundedSource.validate();
- }
-
- @Override
- public Coder<V> getDefaultOutputCoder() {
- return outputCoder;
- }
-
- private class TransformingReader extends BoundedReader<V> {
- private final BoundedReader<T> boundedReader;
-
- private TransformingReader(BoundedReader<T> boundedReader) {
- this.boundedReader = checkNotNull(boundedReader, "boundedReader");
- }
-
- @Override
- public synchronized BoundedSource<V> getCurrentSource() {
- return new TransformingSource<>(boundedReader.getCurrentSource(), function, outputCoder);
- }
-
- @Override
- public boolean start() throws IOException {
- return boundedReader.start();
- }
-
- @Override
- public boolean advance() throws IOException {
- return boundedReader.advance();
- }
-
- @Override
- public V getCurrent() throws NoSuchElementException {
- T current = boundedReader.getCurrent();
- return function.apply(current);
- }
-
- @Override
- public void close() throws IOException {
- boundedReader.close();
- }
-
- @Override
- public synchronized BoundedSource<V> splitAtFraction(double fraction) {
- BoundedSource<T> split = boundedReader.splitAtFraction(fraction);
- return split == null ? null : new TransformingSource<>(split, function, outputCoder);
- }
-
- @Override
- public Double getFractionConsumed() {
- return boundedReader.getFractionConsumed();
- }
-
- @Override
- public Instant getCurrentTimestamp() throws NoSuchElementException {
- return boundedReader.getCurrentTimestamp();
- }
- }
- }
-
- private static String getExtractJobId(ValueProvider<String> jobIdToken) {
+ static String getExtractJobId(ValueProvider<String> jobIdToken) {
return jobIdToken.get() + "-extract";
}
- private static String getExtractDestinationUri(String extractDestinationDir) {
+ static String getExtractDestinationUri(String extractDestinationDir) {
return String.format("%s/%s", extractDestinationDir, "*.avro");
}
- private static List<String> getExtractFilePaths(String extractDestinationDir, Job extractJob)
+ static List<String> getExtractFilePaths(String extractDestinationDir, Job extractJob)
throws IOException {
JobStatistics jobStats = extractJob.getStatistics();
List<Long> counts = jobStats.getExtract().getDestinationUriFileCounts();
@@ -1538,7 +864,8 @@ public class BigQueryIO {
}
/**
- * Writes to the given table, specified in the format described in {@link #parseTableSpec}.
+ * Writes to the given table, specified in the format described in
+ * {@link BigQueryHelpers#parseTableSpec}.
*/
public Write<T> to(String tableSpec) {
return to(StaticValueProvider.of(tableSpec));
@@ -1546,7 +873,7 @@ public class BigQueryIO {
/** Writes to the given table, specified as a {@link TableReference}. */
public Write<T> to(TableReference table) {
- return to(StaticValueProvider.of(toTableSpec(table)));
+ return to(StaticValueProvider.of(BigQueryHelpers.toTableSpec(table)));
}
/** Same as {@link #to(String)}, but with a {@link ValueProvider}. */
@@ -1596,7 +923,7 @@ public class BigQueryIO {
@Override
public TableReference apply(ValueInSingleWindow<T> value) {
- return parseTableSpec(tableSpecFunction.apply(value));
+ return BigQueryHelpers.parseTableSpec(tableSpecFunction.apply(value));
}
}
@@ -1609,7 +936,7 @@ public class BigQueryIO {
*/
public Write<T> withSchema(TableSchema schema) {
return toBuilder()
- .setJsonSchema(StaticValueProvider.of(toJsonString(schema)))
+ .setJsonSchema(StaticValueProvider.of(BigQueryHelpers.toJsonString(schema)))
.build();
}
@@ -1655,7 +982,7 @@ public class BigQueryIO {
checkState(
datasetService.isTableEmpty(tableRef),
"BigQuery table is not empty: %s.",
- BigQueryIO.toTableSpec(tableRef));
+ BigQueryHelpers.toTableSpec(tableRef));
}
} catch (IOException | InterruptedException e) {
if (e instanceof InterruptedException) {
@@ -1663,7 +990,7 @@ public class BigQueryIO {
}
throw new RuntimeException(
"unable to confirm BigQuery table emptiness for table "
- + BigQueryIO.toTableSpec(tableRef), e);
+ + BigQueryHelpers.toTableSpec(tableRef), e);
}
}
@@ -1758,7 +1085,7 @@ public class BigQueryIO {
ValueProvider<TableReference> table = getTableWithDefaultProject(options);
- final String stepUuid = randomUUIDString();
+ final String stepUuid = BigQueryHelpers.randomUUIDString();
String tempLocation = options.getTempLocation();
String tempFilePrefix;
@@ -1953,7 +1280,7 @@ public class BigQueryIO {
/** Returns the table schema. */
public TableSchema getSchema() {
- return fromJsonString(
+ return BigQueryHelpers.fromJsonString(
getJsonSchema() == null ? null : getJsonSchema().get(), TableSchema.class);
}
@@ -1979,7 +1306,7 @@ public class BigQueryIO {
TableReference tableRef = table.get();
tableRef.setProjectId(bqOptions.getProject());
return NestedValueProvider.of(StaticValueProvider.of(
- toJsonString(tableRef)), new JsonTableRefToTableRef());
+ BigQueryHelpers.toJsonString(tableRef)), new JsonTableRefToTableRef());
}
return table;
}
@@ -2128,7 +1455,8 @@ public class BigQueryIO {
List<String> partition = Lists.newArrayList(c.element().getValue()).get(0);
String jobIdPrefix = String.format(
c.sideInput(jobIdToken) + "_%05d", c.element().getKey());
- TableReference ref = fromJsonString(jsonTableRef.get(), TableReference.class);
+ TableReference ref = BigQueryHelpers.fromJsonString(jsonTableRef.get(),
+ TableReference.class);
if (!singlePartition) {
ref.setTableId(jobIdPrefix);
}
@@ -2138,13 +1466,13 @@ public class BigQueryIO {
bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
jobIdPrefix,
ref,
- fromJsonString(
+ BigQueryHelpers.fromJsonString(
jsonSchema == null ? null : jsonSchema.get(), TableSchema.class),
partition,
writeDisposition,
createDisposition,
tableDescription);
- c.output(toJsonString(ref));
+ c.output(BigQueryHelpers.toJsonString(ref));
removeTemporaryFiles(c.getPipelineOptions(), tempFilePrefix, partition);
}
@@ -2176,7 +1504,7 @@ public class BigQueryIO {
.setJobId(jobId);
jobService.startLoadJob(jobRef, loadConfig);
Job loadJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
- Status jobStatus = parseStatus(loadJob);
+ Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
switch (jobStatus) {
case SUCCEEDED:
if (tableDescription != null) {
@@ -2185,14 +1513,15 @@ public class BigQueryIO {
return;
case UNKNOWN:
throw new RuntimeException(String.format(
- "UNKNOWN status of load job [%s]: %s.", jobId, jobToPrettyString(loadJob)));
+ "UNKNOWN status of load job [%s]: %s.", jobId,
+ BigQueryHelpers.jobToPrettyString(loadJob)));
case FAILED:
lastFailedLoadJob = loadJob;
continue;
default:
throw new IllegalStateException(String.format(
"Unexpected status [%s] of load job: %s.",
- jobStatus, jobToPrettyString(loadJob)));
+ jobStatus, BigQueryHelpers.jobToPrettyString(loadJob)));
}
}
throw new RuntimeException(String.format(
@@ -2200,7 +1529,7 @@ public class BigQueryIO {
+ "reached max retries: %d, last failed load job: %s.",
jobIdPrefix,
Write.MAX_RETRY_JOBS,
- jobToPrettyString(lastFailedLoadJob)));
+ BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
}
static void removeTemporaryFiles(
@@ -2281,13 +1610,13 @@ public class BigQueryIO {
List<TableReference> tempTables = Lists.newArrayList();
for (String table : tempTablesJson) {
- tempTables.add(fromJsonString(table, TableReference.class));
+ tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
}
copy(
bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
c.sideInput(jobIdToken),
- fromJsonString(jsonTableRef.get(), TableReference.class),
+ BigQueryHelpers.fromJsonString(jsonTableRef.get(), TableReference.class),
tempTables,
writeDisposition,
createDisposition,
@@ -2322,7 +1651,7 @@ public class BigQueryIO {
.setJobId(jobId);
jobService.startCopyJob(jobRef, copyConfig);
Job copyJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
- Status jobStatus = parseStatus(copyJob);
+ Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
switch (jobStatus) {
case SUCCEEDED:
if (tableDescription != null) {
@@ -2331,14 +1660,15 @@ public class BigQueryIO {
return;
case UNKNOWN:
throw new RuntimeException(String.format(
- "UNKNOWN status of copy job [%s]: %s.", jobId, jobToPrettyString(copyJob)));
+ "UNKNOWN status of copy job [%s]: %s.", jobId,
+ BigQueryHelpers.jobToPrettyString(copyJob)));
case FAILED:
lastFailedCopyJob = copyJob;
continue;
default:
throw new IllegalStateException(String.format(
"Unexpected status [%s] of load job: %s.",
- jobStatus, jobToPrettyString(copyJob)));
+ jobStatus, BigQueryHelpers.jobToPrettyString(copyJob)));
}
}
throw new RuntimeException(String.format(
@@ -2346,17 +1676,17 @@ public class BigQueryIO {
+ "reached max retries: %d, last failed copy job: %s.",
jobIdPrefix,
Write.MAX_RETRY_JOBS,
- jobToPrettyString(lastFailedCopyJob)));
+ BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
}
static void removeTemporaryTables(DatasetService tableService,
List<TableReference> tempTables) {
for (TableReference tableRef : tempTables) {
try {
- LOG.debug("Deleting table {}", toJsonString(tableRef));
+ LOG.debug("Deleting table {}", BigQueryHelpers.toJsonString(tableRef));
tableService.deleteTable(tableRef);
} catch (Exception e) {
- LOG.warn("Failed to delete the table {}", toJsonString(tableRef), e);
+ LOG.warn("Failed to delete the table {}", BigQueryHelpers.toJsonString(tableRef), e);
}
}
}
@@ -2376,14 +1706,6 @@ public class BigQueryIO {
}
}
- private static String jobToPrettyString(@Nullable Job job) throws IOException {
- return job == null ? "null" : job.toPrettyString();
- }
-
- private static String statusToPrettyString(@Nullable JobStatus status) throws IOException {
- return status == null ? "Unknown status: null." : status.toPrettyString();
- }
-
private static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
try {
datasetService.getDataset(table.getProjectId(), table.getDatasetId());
@@ -2391,14 +1713,14 @@ public class BigQueryIO {
ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
throw new IllegalArgumentException(
- String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", BigQueryIO.toTableSpec(table)),
+ String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", BigQueryHelpers.toTableSpec(table)),
e);
} else if (e instanceof RuntimeException) {
throw (RuntimeException) e;
} else {
throw new RuntimeException(
String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "dataset",
- BigQueryIO.toTableSpec(table)),
+ BigQueryHelpers.toTableSpec(table)),
e);
}
}
@@ -2411,13 +1733,14 @@ public class BigQueryIO {
ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
throw new IllegalArgumentException(
- String.format(RESOURCE_NOT_FOUND_ERROR, "table", BigQueryIO.toTableSpec(table)), e);
+ String.format(RESOURCE_NOT_FOUND_ERROR, "table", BigQueryHelpers.toTableSpec(table)),
+ e);
} else if (e instanceof RuntimeException) {
throw (RuntimeException) e;
} else {
throw new RuntimeException(
String.format(UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "table",
- BigQueryIO.toTableSpec(table)),
+ BigQueryHelpers.toTableSpec(table)),
e);
}
}
@@ -2492,8 +1815,9 @@ public class BigQueryIO {
@ProcessElement
public void processElement(ProcessContext context) {
String tableSpec = context.element().getKey().getKey();
- List<TableRow> rows = getOrCreateMapListValue(tableRows, tableSpec);
- List<String> uniqueIds = getOrCreateMapListValue(uniqueIdsForTableRows, tableSpec);
+ List<TableRow> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
+ List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows,
+ tableSpec);
rows.add(context.element().getValue().tableRow);
uniqueIds.add(context.element().getValue().uniqueId);
@@ -2526,7 +1850,7 @@ public class BigQueryIO {
public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
throws InterruptedException, IOException {
- TableReference tableReference = parseTableSpec(tableSpec);
+ TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec);
if (createDisposition != createDisposition.CREATE_NEVER
&& !createdTables.contains(tableSpec)) {
synchronized (createdTables) {
@@ -2723,7 +2047,7 @@ public class BigQueryIO {
TableReference tableRef = table.get()
.setProjectId(options.as(BigQueryOptions.class).getProject());
table = NestedValueProvider.of(
- StaticValueProvider.of(toJsonString(tableRef)),
+ StaticValueProvider.of(BigQueryHelpers.toJsonString(tableRef)),
new JsonTableRefToTableRef());
}
this.tableSpec = NestedValueProvider.of(table, new TableRefToTableSpec());
@@ -2779,7 +2103,7 @@ public class BigQueryIO {
if (table.getProjectId() == null) {
table.setProjectId(options.getProject());
}
- return toTableSpec(table);
+ return BigQueryHelpers.toTableSpec(table);
}
}
}
@@ -2858,68 +2182,9 @@ public class BigQueryIO {
UNKNOWN,
}
- private static Status parseStatus(@Nullable Job job) {
- if (job == null) {
- return Status.UNKNOWN;
- }
- JobStatus status = job.getStatus();
- if (status.getErrorResult() != null) {
- return Status.FAILED;
- } else if (status.getErrors() != null && !status.getErrors().isEmpty()) {
- return Status.FAILED;
- } else {
- return Status.SUCCEEDED;
- }
- }
-
- @VisibleForTesting
- static String toJsonString(Object item) {
- if (item == null) {
- return null;
- }
- try {
- return JSON_FACTORY.toString(item);
- } catch (IOException e) {
- throw new RuntimeException(
- String.format("Cannot serialize %s to a JSON string.", item.getClass().getSimpleName()),
- e);
- }
- }
-
- @VisibleForTesting
- static <T> T fromJsonString(String json, Class<T> clazz) {
- if (json == null) {
- return null;
- }
- try {
- return JSON_FACTORY.fromString(json, clazz);
- } catch (IOException e) {
- throw new RuntimeException(
- String.format("Cannot deserialize %s from a JSON string: %s.", clazz, json),
- e);
- }
- }
-
- /**
- * Returns a randomUUID string.
- *
- * <p>{@code '-'} is removed because BigQuery doesn't allow it in dataset id.
- */
- private static String randomUUIDString() {
- return UUID.randomUUID().toString().replaceAll("-", "");
- }
-
/////////////////////////////////////////////////////////////////////////////
/** Disallow construction of utility class. */
private BigQueryIO() {}
- private static <K, V> List<V> getOrCreateMapListValue(Map<K, List<V>> map, K key) {
- List<V> value = map.get(key);
- if (value == null) {
- value = new ArrayList<>();
- map.put(key, value);
- }
- return value;
- }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
new file mode 100644
index 0000000..a909957
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
@@ -0,0 +1,186 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationQuery;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.JobStatistics;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToProjectId;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+
+/**
+ * A {@link BigQuerySourceBase} for querying BigQuery tables.
+ */
+@VisibleForTesting
+class BigQueryQuerySource extends BigQuerySourceBase {
+
+ static BigQueryQuerySource create(
+ ValueProvider<String> jobIdToken,
+ ValueProvider<String> query,
+ ValueProvider<TableReference> queryTempTableRef,
+ Boolean flattenResults,
+ Boolean useLegacySql,
+ String extractDestinationDir,
+ BigQueryServices bqServices) {
+ return new BigQueryQuerySource(
+ jobIdToken,
+ query,
+ queryTempTableRef,
+ flattenResults,
+ useLegacySql,
+ extractDestinationDir,
+ bqServices);
+ }
+
+ private final ValueProvider<String> query;
+ private final ValueProvider<String> jsonQueryTempTable;
+ private final Boolean flattenResults;
+ private final Boolean useLegacySql;
+ private transient AtomicReference<JobStatistics> dryRunJobStats;
+
+ private BigQueryQuerySource(
+ ValueProvider<String> jobIdToken,
+ ValueProvider<String> query,
+ ValueProvider<TableReference> queryTempTableRef,
+ Boolean flattenResults,
+ Boolean useLegacySql,
+ String extractDestinationDir,
+ BigQueryServices bqServices) {
+ super(jobIdToken, extractDestinationDir, bqServices,
+ NestedValueProvider.of(
+ checkNotNull(queryTempTableRef, "queryTempTableRef"), new TableRefToProjectId()));
+ this.query = checkNotNull(query, "query");
+ this.jsonQueryTempTable = NestedValueProvider.of(
+ queryTempTableRef, new TableRefToJson());
+ this.flattenResults = checkNotNull(flattenResults, "flattenResults");
+ this.useLegacySql = checkNotNull(useLegacySql, "useLegacySql");
+ this.dryRunJobStats = new AtomicReference<>();
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+ BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+ return dryRunQueryIfNeeded(bqOptions).getTotalBytesProcessed();
+ }
+
+ @Override
+ public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
+ BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+ return new BigQueryReader(this, bqServices.getReaderFromQuery(
+ bqOptions, executingProject.get(), createBasicQueryConfig()));
+ }
+
+ @Override
+ protected TableReference getTableToExtract(BigQueryOptions bqOptions)
+ throws IOException, InterruptedException {
+ // 1. Find the location of the query.
+ String location = null;
+ List<TableReference> referencedTables =
+ dryRunQueryIfNeeded(bqOptions).getQuery().getReferencedTables();
+ DatasetService tableService = bqServices.getDatasetService(bqOptions);
+ if (referencedTables != null && !referencedTables.isEmpty()) {
+ TableReference queryTable = referencedTables.get(0);
+ location = tableService.getTable(queryTable).getLocation();
+ }
+
+ // 2. Create the temporary dataset in the query location.
+ TableReference tableToExtract =
+ BigQueryIO.JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
+ tableService.createDataset(
+ tableToExtract.getProjectId(),
+ tableToExtract.getDatasetId(),
+ location,
+ "Dataset for BigQuery query job temporary table");
+
+ // 3. Execute the query.
+ String queryJobId = jobIdToken.get() + "-query";
+ executeQuery(
+ executingProject.get(),
+ queryJobId,
+ tableToExtract,
+ bqServices.getJobService(bqOptions));
+ return tableToExtract;
+ }
+
+ @Override
+ protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
+ checkState(jsonQueryTempTable.isAccessible());
+ TableReference tableToRemove =
+ BigQueryIO.JSON_FACTORY.fromString(jsonQueryTempTable.get(), TableReference.class);
+
+ DatasetService tableService = bqServices.getDatasetService(bqOptions);
+ tableService.deleteTable(tableToRemove);
+ tableService.deleteDataset(tableToRemove.getProjectId(), tableToRemove.getDatasetId());
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("query", query));
+ }
+
+ private synchronized JobStatistics dryRunQueryIfNeeded(BigQueryOptions bqOptions)
+ throws InterruptedException, IOException {
+ if (dryRunJobStats.get() == null) {
+ JobStatistics jobStats = bqServices.getJobService(bqOptions).dryRunQuery(
+ executingProject.get(), createBasicQueryConfig());
+ dryRunJobStats.compareAndSet(null, jobStats);
+ }
+ return dryRunJobStats.get();
+ }
+
+ private void executeQuery(
+ String executingProject,
+ String jobId,
+ TableReference destinationTable,
+ JobService jobService) throws IOException, InterruptedException {
+ JobReference jobRef = new JobReference()
+ .setProjectId(executingProject)
+ .setJobId(jobId);
+
+ JobConfigurationQuery queryConfig = createBasicQueryConfig()
+ .setAllowLargeResults(true)
+ .setCreateDisposition("CREATE_IF_NEEDED")
+ .setDestinationTable(destinationTable)
+ .setPriority("BATCH")
+ .setWriteDisposition("WRITE_EMPTY");
+
+ jobService.startQueryJob(jobRef, queryConfig);
+ Job job = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
+ if (BigQueryHelpers.parseStatus(job) != Status.SUCCEEDED) {
+ throw new IOException(String.format(
+ "Query job %s failed, status: %s.", jobId,
+ BigQueryHelpers.statusToPrettyString(job.getStatus())));
+ }
+ }
+
+ private JobConfigurationQuery createBasicQueryConfig() {
+ return new JobConfigurationQuery()
+ .setFlattenResults(flattenResults)
+ .setQuery(query.get())
+ .setUseLegacySql(useLegacySql);
+ }
+
+ private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException {
+ in.defaultReadObject();
+ dryRunJobStats = new AtomicReference<>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index 15ca262..c8e6ed8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -444,7 +444,7 @@ class BigQueryServicesImpl implements BigQueryServices {
@Override
public void createTable(Table table) throws InterruptedException, IOException {
LOG.info("Trying to create BigQuery table: {}",
- BigQueryIO.toTableSpec(table.getTableReference()));
+ BigQueryHelpers.toTableSpec(table.getTableReference()));
BackOff backoff =
new ExponentialBackOff.Builder()
.setMaxElapsedTimeMillis(RETRY_CREATE_TABLE_DURATION_MILLIS)
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
new file mode 100644
index 0000000..ff50e6d
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -0,0 +1,178 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationExtract;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.TableRowJsonCoder;
+import org.apache.beam.sdk.io.AvroSource;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An abstract {@link BoundedSource} to read a table from BigQuery.
+ *
+ * <p>This source uses a BigQuery export job to take a snapshot of the table on GCS, and then
+ * reads in parallel from each produced file. It is implemented by {@link BigQueryTableSource},
+ * and {@link BigQueryQuerySource}, depending on the configuration of the read.
+ * Specifically,
+ * <ul>
+ * <li>{@link BigQueryTableSource} is for reading BigQuery tables</li>
+ * <li>{@link BigQueryQuerySource} is for querying BigQuery tables</li>
+ * </ul>
+ * ...
+ */
+abstract class BigQuerySourceBase extends BoundedSource<TableRow> {
+ private static final Logger LOG = LoggerFactory.getLogger(BigQuerySourceBase.class);
+
+ // The maximum number of retries to poll a BigQuery job.
+ protected static final int JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
+
+ protected final ValueProvider<String> jobIdToken;
+ protected final String extractDestinationDir;
+ protected final BigQueryServices bqServices;
+ protected final ValueProvider<String> executingProject;
+
+ BigQuerySourceBase(
+ ValueProvider<String> jobIdToken,
+ String extractDestinationDir,
+ BigQueryServices bqServices,
+ ValueProvider<String> executingProject) {
+ this.jobIdToken = checkNotNull(jobIdToken, "jobIdToken");
+ this.extractDestinationDir = checkNotNull(extractDestinationDir, "extractDestinationDir");
+ this.bqServices = checkNotNull(bqServices, "bqServices");
+ this.executingProject = checkNotNull(executingProject, "executingProject");
+ }
+
+ @Override
+ public List<BoundedSource<TableRow>> splitIntoBundles(
+ long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+ BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+ TableReference tableToExtract = getTableToExtract(bqOptions);
+ JobService jobService = bqServices.getJobService(bqOptions);
+ String extractJobId = BigQueryIO.getExtractJobId(jobIdToken);
+ List<String> tempFiles = executeExtract(extractJobId, tableToExtract, jobService);
+
+ TableSchema tableSchema = bqServices.getDatasetService(bqOptions)
+ .getTable(tableToExtract).getSchema();
+
+ cleanupTempResource(bqOptions);
+ return createSources(tempFiles, tableSchema);
+ }
+
+ protected abstract TableReference getTableToExtract(BigQueryOptions bqOptions) throws Exception;
+
+ protected abstract void cleanupTempResource(BigQueryOptions bqOptions) throws Exception;
+
+ @Override
+ public void validate() {
+ // Do nothing, validation is done in BigQuery.Read.
+ }
+
+ @Override
+ public Coder<TableRow> getDefaultOutputCoder() {
+ return TableRowJsonCoder.of();
+ }
+
+ private List<String> executeExtract(
+ String jobId, TableReference table, JobService jobService)
+ throws InterruptedException, IOException {
+ JobReference jobRef = new JobReference()
+ .setProjectId(executingProject.get())
+ .setJobId(jobId);
+
+ String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir);
+ JobConfigurationExtract extract = new JobConfigurationExtract()
+ .setSourceTable(table)
+ .setDestinationFormat("AVRO")
+ .setDestinationUris(ImmutableList.of(destinationUri));
+
+ LOG.info("Starting BigQuery extract job: {}", jobId);
+ jobService.startExtractJob(jobRef, extract);
+ Job extractJob =
+ jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES);
+ if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) {
+ throw new IOException(String.format(
+ "Extract job %s failed, status: %s.",
+ extractJob.getJobReference().getJobId(),
+ BigQueryHelpers.statusToPrettyString(extractJob.getStatus())));
+ }
+
+ List<String> tempFiles = BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob);
+ return ImmutableList.copyOf(tempFiles);
+ }
+
+ private List<BoundedSource<TableRow>> createSources(
+ List<String> files, TableSchema tableSchema) throws IOException, InterruptedException {
+ final String jsonSchema = BigQueryIO.JSON_FACTORY.toString(tableSchema);
+
+ SerializableFunction<GenericRecord, TableRow> function =
+ new SerializableFunction<GenericRecord, TableRow>() {
+ @Override
+ public TableRow apply(GenericRecord input) {
+ return BigQueryAvroUtils.convertGenericRecordToTableRow(
+ input, BigQueryHelpers.fromJsonString(jsonSchema, TableSchema.class));
+ }};
+
+ List<BoundedSource<TableRow>> avroSources = Lists.newArrayList();
+ for (String fileName : files) {
+ avroSources.add(new TransformingSource<>(
+ AvroSource.from(fileName), function, getDefaultOutputCoder()));
+ }
+ return ImmutableList.copyOf(avroSources);
+ }
+
+ protected static class BigQueryReader extends BoundedReader<TableRow> {
+ private final BigQuerySourceBase source;
+ private final BigQueryServices.BigQueryJsonReader reader;
+
+ BigQueryReader(
+ BigQuerySourceBase source, BigQueryServices.BigQueryJsonReader reader) {
+ this.source = source;
+ this.reader = reader;
+ }
+
+ @Override
+ public BoundedSource<TableRow> getCurrentSource() {
+ return source;
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ return reader.start();
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ return reader.advance();
+ }
+
+ @Override
+ public TableRow getCurrent() throws NoSuchElementException {
+ return reader.getCurrent();
+ }
+
+ @Override
+ public void close() throws IOException {
+ reader.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
new file mode 100644
index 0000000..aae0faa
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
@@ -0,0 +1,86 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+
+/**
+ * A {@link BigQuerySourceBase} for reading BigQuery tables.
+ */
+@VisibleForTesting
+class BigQueryTableSource extends BigQuerySourceBase {
+
+ static BigQueryTableSource create(
+ ValueProvider<String> jobIdToken,
+ ValueProvider<TableReference> table,
+ String extractDestinationDir,
+ BigQueryServices bqServices,
+ ValueProvider<String> executingProject) {
+ return new BigQueryTableSource(
+ jobIdToken, table, extractDestinationDir, bqServices, executingProject);
+ }
+
+ private final ValueProvider<String> jsonTable;
+ private final AtomicReference<Long> tableSizeBytes;
+
+ private BigQueryTableSource(
+ ValueProvider<String> jobIdToken,
+ ValueProvider<TableReference> table,
+ String extractDestinationDir,
+ BigQueryServices bqServices,
+ ValueProvider<String> executingProject) {
+ super(jobIdToken, extractDestinationDir, bqServices, executingProject);
+ this.jsonTable = NestedValueProvider.of(checkNotNull(table, "table"), new TableRefToJson());
+ this.tableSizeBytes = new AtomicReference<>();
+ }
+
+ @Override
+ protected TableReference getTableToExtract(BigQueryOptions bqOptions) throws IOException {
+ checkState(jsonTable.isAccessible());
+ return BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(), TableReference.class);
+ }
+
+ @Override
+ public BoundedReader<TableRow> createReader(PipelineOptions options) throws IOException {
+ BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+ checkState(jsonTable.isAccessible());
+ TableReference tableRef = BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(),
+ TableReference.class);
+ return new BigQueryReader(this, bqServices.getReaderFromTable(bqOptions, tableRef));
+ }
+
+ @Override
+ public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+ if (tableSizeBytes.get() == null) {
+ TableReference table = BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(),
+ TableReference.class);
+
+ Long numBytes = bqServices.getDatasetService(options.as(BigQueryOptions.class))
+ .getTable(table).getNumBytes();
+ tableSizeBytes.compareAndSet(null, numBytes);
+ }
+ return tableSizeBytes.get();
+ }
+
+ @Override
+ protected void cleanupTempResource(BigQueryOptions bqOptions) throws Exception {
+ // Do nothing.
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+ builder.add(DisplayData.item("table", jsonTable));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
new file mode 100644
index 0000000..612afbe
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
@@ -0,0 +1,66 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+
+/**
+ * A {@link PTransform} that invokes {@link CleanupOperation} after the input {@link PCollection}
+ * has been processed.
+ */
+@VisibleForTesting
+class PassThroughThenCleanup<T> extends PTransform<PCollection<T>, PCollection<T>> {
+
+ private CleanupOperation cleanupOperation;
+
+ PassThroughThenCleanup(CleanupOperation cleanupOperation) {
+ this.cleanupOperation = cleanupOperation;
+ }
+
+ @Override
+ public PCollection<T> expand(PCollection<T> input) {
+ TupleTag<T> mainOutput = new TupleTag<>();
+ TupleTag<Void> cleanupSignal = new TupleTag<>();
+ PCollectionTuple outputs = input.apply(ParDo.of(new IdentityFn<T>())
+ .withOutputTags(mainOutput, TupleTagList.of(cleanupSignal)));
+
+ PCollectionView<Void> cleanupSignalView = outputs.get(cleanupSignal)
+ .setCoder(VoidCoder.of())
+ .apply(View.<Void>asSingleton().withDefaultValue(null));
+
+ input.getPipeline()
+ .apply("Create(CleanupOperation)", Create.of(cleanupOperation))
+ .apply("Cleanup", ParDo.of(
+ new DoFn<CleanupOperation, Void>() {
+ @ProcessElement
+ public void processElement(ProcessContext c)
+ throws Exception {
+ c.element().cleanup(c.getPipelineOptions());
+ }
+ }).withSideInputs(cleanupSignalView));
+
+ return outputs.get(mainOutput);
+ }
+
+ private static class IdentityFn<T> extends DoFn<T, T> {
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ c.output(c.element());
+ }
+ }
+
+ abstract static class CleanupOperation implements Serializable {
+ abstract void cleanup(PipelineOptions options) throws Exception;
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
new file mode 100644
index 0000000..a86adfb
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
@@ -0,0 +1,118 @@
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.BoundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.joda.time.Instant;
+
+/**
+ * A {@link BoundedSource} that reads from {@code BoundedSource<T>}
+ * and transforms elements to type {@code V}.
+*/
+@VisibleForTesting
+class TransformingSource<T, V> extends BoundedSource<V> {
+ private final BoundedSource<T> boundedSource;
+ private final SerializableFunction<T, V> function;
+ private final Coder<V> outputCoder;
+
+ TransformingSource(
+ BoundedSource<T> boundedSource,
+ SerializableFunction<T, V> function,
+ Coder<V> outputCoder) {
+ this.boundedSource = checkNotNull(boundedSource, "boundedSource");
+ this.function = checkNotNull(function, "function");
+ this.outputCoder = checkNotNull(outputCoder, "outputCoder");
+ }
+
+ @Override
+ public List<? extends BoundedSource<V>> splitIntoBundles(
+ long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
+ return Lists.transform(
+ boundedSource.splitIntoBundles(desiredBundleSizeBytes, options),
+ new Function<BoundedSource<T>, BoundedSource<V>>() {
+ @Override
+ public BoundedSource<V> apply(BoundedSource<T> input) {
+ return new TransformingSource<>(input, function, outputCoder);
+ }
+ });
+ }
+
+ @Override
+ public long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
+ return boundedSource.getEstimatedSizeBytes(options);
+ }
+
+ @Override
+ public BoundedReader<V> createReader(PipelineOptions options) throws IOException {
+ return new TransformingReader(boundedSource.createReader(options));
+ }
+
+ @Override
+ public void validate() {
+ boundedSource.validate();
+ }
+
+ @Override
+ public Coder<V> getDefaultOutputCoder() {
+ return outputCoder;
+ }
+
+ private class TransformingReader extends BoundedReader<V> {
+ private final BoundedReader<T> boundedReader;
+
+ private TransformingReader(BoundedReader<T> boundedReader) {
+ this.boundedReader = checkNotNull(boundedReader, "boundedReader");
+ }
+
+ @Override
+ public synchronized BoundedSource<V> getCurrentSource() {
+ return new TransformingSource<>(boundedReader.getCurrentSource(), function, outputCoder);
+ }
+
+ @Override
+ public boolean start() throws IOException {
+ return boundedReader.start();
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ return boundedReader.advance();
+ }
+
+ @Override
+ public V getCurrent() throws NoSuchElementException {
+ T current = boundedReader.getCurrent();
+ return function.apply(current);
+ }
+
+ @Override
+ public void close() throws IOException {
+ boundedReader.close();
+ }
+
+ @Override
+ public synchronized BoundedSource<V> splitAtFraction(double fraction) {
+ BoundedSource<T> split = boundedReader.splitAtFraction(fraction);
+ return split == null ? null : new TransformingSource<>(split, function, outputCoder);
+ }
+
+ @Override
+ public Double getFractionConsumed() {
+ return boundedReader.getFractionConsumed();
+ }
+
+ @Override
+ public Instant getCurrentTimestamp() throws NoSuchElementException {
+ return boundedReader.getCurrentTimestamp();
+ }
+ }
+}
[3/6] beam git commit: Refactor BigQueryIO.Write helper transforms
into separate files.
Posted by jk...@apache.org.
Refactor BigQueryIO.Write helper transforms into separate files.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d6ef0104
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d6ef0104
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d6ef0104
Branch: refs/heads/master
Commit: d6ef0104d5b8e1076cea2a9d08ad14ba0e8e84f1
Parents: 2cc2e81
Author: Reuven Lax <re...@google.com>
Authored: Fri Mar 17 15:30:53 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:15 2017 -0700
----------------------------------------------------------------------
.../sdk/io/gcp/bigquery/BatchLoadBigQuery.java | 182 +++
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 64 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 1063 +-----------------
.../io/gcp/bigquery/BigQueryQuerySource.java | 21 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 20 +-
.../io/gcp/bigquery/BigQueryTableSource.java | 18 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 18 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 44 +
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 87 ++
.../sdk/io/gcp/bigquery/StreamWithDeDup.java | 98 ++
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 186 +++
.../beam/sdk/io/gcp/bigquery/TableRowInfo.java | 34 +
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 68 ++
.../sdk/io/gcp/bigquery/TableRowWriter.java | 84 ++
.../gcp/bigquery/TagWithUniqueIdsAndTable.java | 135 +++
.../sdk/io/gcp/bigquery/TransformingSource.java | 18 +
.../beam/sdk/io/gcp/bigquery/WriteBundles.java | 82 ++
.../sdk/io/gcp/bigquery/WritePartition.java | 79 ++
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 180 +++
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 213 ++++
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 39 +-
21 files changed, 1653 insertions(+), 1080 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java
new file mode 100644
index 0000000..75b1cc7
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadBigQuery.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import java.io.IOException;
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.View;
+import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.IOChannelFactory;
+import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.PDone;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+/**
+ * PTransform that uses BigQuery batch-load jobs to write a PCollection to BigQuery.
+ */
+class BatchLoadBigQuery<T> extends PTransform<PCollection<T>, PDone> {
+ BigQueryIO.Write<T> write;
+
+ BatchLoadBigQuery(BigQueryIO.Write<T> write) {
+ this.write = write;
+ }
+
+ @Override
+ public PDone expand(PCollection<T> input) {
+ Pipeline p = input.getPipeline();
+ BigQueryOptions options = p.getOptions().as(BigQueryOptions.class);
+ ValueProvider<TableReference> table = write.getTableWithDefaultProject(options);
+
+ final String stepUuid = BigQueryHelpers.randomUUIDString();
+
+ String tempLocation = options.getTempLocation();
+ String tempFilePrefix;
+ try {
+ IOChannelFactory factory = IOChannelUtils.getFactory(tempLocation);
+ tempFilePrefix = factory.resolve(
+ factory.resolve(tempLocation, "BigQueryWriteTemp"),
+ stepUuid);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format("Failed to resolve BigQuery temp location in %s", tempLocation),
+ e);
+ }
+
+ // Create a singleton job ID token at execution time.
+ PCollection<String> singleton = p.apply("Create", Create.of(tempFilePrefix));
+ PCollectionView<String> jobIdTokenView = p
+ .apply("TriggerIdCreation", Create.of("ignored"))
+ .apply("CreateJobId", MapElements.via(
+ new SimpleFunction<String, String>() {
+ @Override
+ public String apply(String input) {
+ return stepUuid;
+ }
+ }))
+ .apply(View.<String>asSingleton());
+
+ PCollection<T> typedInputInGlobalWindow =
+ input.apply(
+ Window.<T>into(new GlobalWindows())
+ .triggering(DefaultTrigger.of())
+ .discardingFiredPanes());
+ // Avoid applying the formatFunction if it is the identity formatter.
+ PCollection<TableRow> inputInGlobalWindow;
+ if (write.getFormatFunction() == BigQueryIO.IDENTITY_FORMATTER) {
+ inputInGlobalWindow = (PCollection<TableRow>) typedInputInGlobalWindow;
+ } else {
+ inputInGlobalWindow = typedInputInGlobalWindow
+ .apply(MapElements.via(write.getFormatFunction())
+ .withOutputType(new TypeDescriptor<TableRow>() {
+ }));
+ }
+
+ // PCollection of filename, file byte size.
+ PCollection<KV<String, Long>> results = inputInGlobalWindow
+ .apply("WriteBundles",
+ ParDo.of(new WriteBundles(tempFilePrefix)));
+
+ TupleTag<KV<Long, List<String>>> multiPartitionsTag =
+ new TupleTag<KV<Long, List<String>>>("multiPartitionsTag") {};
+ TupleTag<KV<Long, List<String>>> singlePartitionTag =
+ new TupleTag<KV<Long, List<String>>>("singlePartitionTag") {};
+
+ // Turn the list of files and record counts in a PCollectionView that can be used as a
+ // side input.
+ PCollectionView<Iterable<KV<String, Long>>> resultsView = results
+ .apply("ResultsView", View.<KV<String, Long>>asIterable());
+ PCollectionTuple partitions = singleton.apply(ParDo
+ .of(new WritePartition(
+ resultsView,
+ multiPartitionsTag,
+ singlePartitionTag))
+ .withSideInputs(resultsView)
+ .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
+
+ // If WriteBundles produced more than MAX_NUM_FILES files or MAX_SIZE_BYTES bytes, then
+ // the import needs to be split into multiple partitions, and those partitions will be
+ // specified in multiPartitionsTag.
+ PCollection<String> tempTables = partitions.get(multiPartitionsTag)
+ .apply("MultiPartitionsGroupByKey", GroupByKey.<Long, List<String>>create())
+ .apply("MultiPartitionsWriteTables", ParDo.of(new WriteTables(
+ false,
+ write.getBigQueryServices(),
+ jobIdTokenView,
+ tempFilePrefix,
+ NestedValueProvider.of(table, new TableRefToJson()),
+ write.getJsonSchema(),
+ WriteDisposition.WRITE_EMPTY,
+ CreateDisposition.CREATE_IF_NEEDED,
+ write.getTableDescription()))
+ .withSideInputs(jobIdTokenView));
+
+ PCollectionView<Iterable<String>> tempTablesView = tempTables
+ .apply("TempTablesView", View.<String>asIterable());
+ singleton.apply(ParDo
+ .of(new WriteRename(
+ write.getBigQueryServices(),
+ jobIdTokenView,
+ NestedValueProvider.of(table, new TableRefToJson()),
+ write.getWriteDisposition(),
+ write.getCreateDisposition(),
+ tempTablesView,
+ write.getTableDescription()))
+ .withSideInputs(tempTablesView, jobIdTokenView));
+
+ // Write single partition to final table
+ partitions.get(singlePartitionTag)
+ .apply("SinglePartitionGroupByKey", GroupByKey.<Long, List<String>>create())
+ .apply("SinglePartitionWriteTables", ParDo.of(new WriteTables(
+ true,
+ write.getBigQueryServices(),
+ jobIdTokenView,
+ tempFilePrefix,
+ NestedValueProvider.of(table, new TableRefToJson()),
+ write.getJsonSchema(),
+ write.getWriteDisposition(),
+ write.getCreateDisposition(),
+ write.getTableDescription()))
+ .withSideInputs(jobIdTokenView));
+
+ return PDone.in(input.getPipeline());
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
index 9fba938..37ff124 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryHelpers.java
@@ -1,5 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.beam.sdk.io.gcp.bigquery;
+import static com.google.common.base.Preconditions.checkState;
+
import com.google.api.services.bigquery.model.Job;
import com.google.api.services.bigquery.model.JobStatus;
import com.google.api.services.bigquery.model.TableReference;
@@ -14,16 +34,34 @@ import java.util.UUID;
import java.util.regex.Matcher;
import javax.annotation.Nullable;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
import org.apache.beam.sdk.transforms.SerializableFunction;
-
/**
* A set of helper functions and classes used by {@link BigQueryIO}.
*/
-public class BigQueryHelpers {
+class BigQueryHelpers {
+ private static final String RESOURCE_NOT_FOUND_ERROR =
+ "BigQuery %1$s not found for table \"%2$s\" . Please create the %1$s before pipeline"
+ + " execution. If the %1$s is created by an earlier stage of the pipeline, this"
+ + " validation can be disabled using #withoutValidation.";
+
+ private static final String UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR =
+ "Unable to confirm BigQuery %1$s presence for table \"%2$s\". If the %1$s is created by"
+ + " an earlier stage of the pipeline, this validation can be disabled using"
+ + " #withoutValidation.";
+
+ /**
+ * Status of a BigQuery job or request.
+ */
+ enum Status {
+ SUCCEEDED,
+ FAILED,
+ UNKNOWN,
+ }
+
@Nullable
/**
* Return a displayable string representation for a {@link TableReference}.
@@ -138,6 +176,26 @@ public class BigQueryHelpers {
return UUID.randomUUID().toString().replaceAll("-", "");
}
+ static void verifyTableNotExistOrEmpty(
+ DatasetService datasetService,
+ TableReference tableRef) {
+ try {
+ if (datasetService.getTable(tableRef) != null) {
+ checkState(
+ datasetService.isTableEmpty(tableRef),
+ "BigQuery table is not empty: %s.",
+ toTableSpec(tableRef));
+ }
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new RuntimeException(
+ "unable to confirm BigQuery table emptiness for table "
+ + toTableSpec(tableRef), e);
+ }
+ }
+
@VisibleForTesting
static class JsonSchemaToTableSchema
implements SerializableFunction<String, TableSchema> {
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 3f2f3e8..4917083 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -18,19 +18,13 @@
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.api.client.json.JsonFactory;
import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobConfigurationLoad;
import com.google.api.services.bigquery.model.JobConfigurationQuery;
-import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
import com.google.api.services.bigquery.model.JobReference;
import com.google.api.services.bigquery.model.JobStatistics;
-import com.google.api.services.bigquery.model.Table;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
@@ -39,54 +33,30 @@ import com.google.cloud.hadoop.util.ApiErrorExtractor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
import com.google.common.io.CountingOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.coders.AtomicCoder;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.Coder.Context;
-import org.apache.beam.sdk.coders.CoderException;
-import org.apache.beam.sdk.coders.KvCoder;
-import org.apache.beam.sdk.coders.StandardCoder;
-import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.coders.TableRowJsonCoder;
-import org.apache.beam.sdk.coders.VarIntCoder;
import org.apache.beam.sdk.coders.VoidCoder;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.BeamJobUuidToBigQueryJobUuid;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreateJsonTableRefFromUuid;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.CreatePerBeamJobUuid;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonTableRefToTableRef;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToTableSpec;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
-import org.apache.beam.sdk.metrics.Counter;
-import org.apache.beam.sdk.metrics.Metrics;
import org.apache.beam.sdk.options.BigQueryOptions;
import org.apache.beam.sdk.options.GcpOptions;
import org.apache.beam.sdk.options.PipelineOptions;
@@ -94,41 +64,19 @@ import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
import org.apache.beam.sdk.runners.PipelineRunner;
-import org.apache.beam.sdk.transforms.Create;
-import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.PTransform;
-import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.SimpleFunction;
-import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.display.DisplayData;
-import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
-import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
-import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
-import org.apache.beam.sdk.transforms.windowing.Window;
-import org.apache.beam.sdk.util.FileIOChannelFactory;
-import org.apache.beam.sdk.util.GcsIOChannelFactory;
-import org.apache.beam.sdk.util.GcsUtil;
import org.apache.beam.sdk.util.GcsUtil.GcsUtilFactory;
import org.apache.beam.sdk.util.IOChannelFactory;
import org.apache.beam.sdk.util.IOChannelUtils;
import org.apache.beam.sdk.util.MimeTypes;
-import org.apache.beam.sdk.util.PropertyNames;
-import org.apache.beam.sdk.util.Reshuffle;
import org.apache.beam.sdk.util.Transport;
import org.apache.beam.sdk.util.gcsfs.GcsPath;
-import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollection.IsBounded;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.PDone;
-import org.apache.beam.sdk.values.TupleTag;
-import org.apache.beam.sdk.values.TupleTagList;
-import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -296,7 +244,7 @@ public class BigQueryIO {
* A formatting function that maps a TableRow to itself. This allows sending a
* {@code PCollection<TableRow>} directly to BigQueryIO.Write.
*/
- private static final SerializableFunction<TableRow, TableRow> IDENTITY_FORMATTER =
+ static final SerializableFunction<TableRow, TableRow> IDENTITY_FORMATTER =
new SerializableFunction<TableRow, TableRow>() {
@Override
public TableRow apply(TableRow input) {
@@ -622,7 +570,7 @@ public class BigQueryIO {
*
* <p>If the table's project is not specified, use the executing project.
*/
- @Nullable private ValueProvider<TableReference> getTableWithDefaultProject(
+ @Nullable ValueProvider<TableReference> getTableWithDefaultProject(
BigQueryOptions bqOptions) {
ValueProvider<TableReference> table = getTableProvider();
if (table == null) {
@@ -753,11 +701,11 @@ public class BigQueryIO {
static final long MAX_SIZE_BYTES = 11 * (1L << 40);
// The maximum number of retry jobs.
- private static final int MAX_RETRY_JOBS = 3;
+ static final int MAX_RETRY_JOBS = 3;
// The maximum number of retries to poll the status of a job.
// It sets to {@code Integer.MAX_VALUE} to block until the BigQuery job finishes.
- private static final int LOAD_JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
+ static final int LOAD_JOB_POLL_MAX_RETRIES = Integer.MAX_VALUE;
@Nullable abstract ValueProvider<String> getJsonTableRef();
@Nullable abstract SerializableFunction<ValueInSingleWindow<T>, TableReference>
@@ -974,26 +922,6 @@ public class BigQueryIO {
return toBuilder().setBigQueryServices(testServices).build();
}
- private static void verifyTableNotExistOrEmpty(
- DatasetService datasetService,
- TableReference tableRef) {
- try {
- if (datasetService.getTable(tableRef) != null) {
- checkState(
- datasetService.isTableEmpty(tableRef),
- "BigQuery table is not empty: %s.",
- BigQueryHelpers.toTableSpec(tableRef));
- }
- } catch (IOException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throw new RuntimeException(
- "unable to confirm BigQuery table emptiness for table "
- + BigQueryHelpers.toTableSpec(tableRef), e);
- }
- }
-
@Override
public void validate(PCollection<T> input) {
BigQueryOptions options = input.getPipeline().getOptions().as(BigQueryOptions.class);
@@ -1030,7 +958,7 @@ public class BigQueryIO {
verifyTablePresence(datasetService, table);
}
if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
- verifyTableNotExistOrEmpty(datasetService, table);
+ BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table);
}
}
@@ -1074,176 +1002,12 @@ public class BigQueryIO {
@Override
public PDone expand(PCollection<T> input) {
- Pipeline p = input.getPipeline();
- BigQueryOptions options = p.getOptions().as(BigQueryOptions.class);
-
// When writing an Unbounded PCollection, or when a tablespec function is defined, we use
// StreamWithDeDup and BigQuery's streaming import API.
if (input.isBounded() == IsBounded.UNBOUNDED || getTableRefFunction() != null) {
return input.apply(new StreamWithDeDup<T>(this));
- }
-
- ValueProvider<TableReference> table = getTableWithDefaultProject(options);
-
- final String stepUuid = BigQueryHelpers.randomUUIDString();
-
- String tempLocation = options.getTempLocation();
- String tempFilePrefix;
- try {
- IOChannelFactory factory = IOChannelUtils.getFactory(tempLocation);
- tempFilePrefix = factory.resolve(
- factory.resolve(tempLocation, "BigQueryWriteTemp"),
- stepUuid);
- } catch (IOException e) {
- throw new RuntimeException(
- String.format("Failed to resolve BigQuery temp location in %s", tempLocation),
- e);
- }
-
- // Create a singleton job ID token at execution time.
- PCollection<String> singleton = p.apply("Create", Create.of(tempFilePrefix));
- PCollectionView<String> jobIdTokenView = p
- .apply("TriggerIdCreation", Create.of("ignored"))
- .apply("CreateJobId", MapElements.via(
- new SimpleFunction<String, String>() {
- @Override
- public String apply(String input) {
- return stepUuid;
- }
- }))
- .apply(View.<String>asSingleton());
-
- PCollection<T> typedInputInGlobalWindow =
- input.apply(
- Window.<T>into(new GlobalWindows())
- .triggering(DefaultTrigger.of())
- .discardingFiredPanes());
- // Avoid applying the formatFunction if it is the identity formatter.
- PCollection<TableRow> inputInGlobalWindow;
- if (getFormatFunction() == IDENTITY_FORMATTER) {
- inputInGlobalWindow = (PCollection<TableRow>) typedInputInGlobalWindow;
} else {
- inputInGlobalWindow = typedInputInGlobalWindow
- .apply(MapElements.via(getFormatFunction())
- .withOutputType(new TypeDescriptor<TableRow>() {
- }));
- }
-
- // PCollection of filename, file byte size.
- PCollection<KV<String, Long>> results = inputInGlobalWindow
- .apply("WriteBundles",
- ParDo.of(new WriteBundles(tempFilePrefix)));
-
- TupleTag<KV<Long, List<String>>> multiPartitionsTag =
- new TupleTag<KV<Long, List<String>>>("multiPartitionsTag") {};
- TupleTag<KV<Long, List<String>>> singlePartitionTag =
- new TupleTag<KV<Long, List<String>>>("singlePartitionTag") {};
-
- // Turn the list of files and record counts in a PCollectionView that can be used as a
- // side input.
- PCollectionView<Iterable<KV<String, Long>>> resultsView = results
- .apply("ResultsView", View.<KV<String, Long>>asIterable());
- PCollectionTuple partitions = singleton.apply(ParDo
- .of(new WritePartition(
- resultsView,
- multiPartitionsTag,
- singlePartitionTag))
- .withSideInputs(resultsView)
- .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag)));
-
- // If WriteBundles produced more than MAX_NUM_FILES files or MAX_SIZE_BYTES bytes, then
- // the import needs to be split into multiple partitions, and those partitions will be
- // specified in multiPartitionsTag.
- PCollection<String> tempTables = partitions.get(multiPartitionsTag)
- .apply("MultiPartitionsGroupByKey", GroupByKey.<Long, List<String>>create())
- .apply("MultiPartitionsWriteTables", ParDo.of(new WriteTables(
- false,
- getBigQueryServices(),
- jobIdTokenView,
- tempFilePrefix,
- NestedValueProvider.of(table, new TableRefToJson()),
- getJsonSchema(),
- WriteDisposition.WRITE_EMPTY,
- CreateDisposition.CREATE_IF_NEEDED,
- getTableDescription()))
- .withSideInputs(jobIdTokenView));
-
- PCollectionView<Iterable<String>> tempTablesView = tempTables
- .apply("TempTablesView", View.<String>asIterable());
- singleton.apply(ParDo
- .of(new WriteRename(
- getBigQueryServices(),
- jobIdTokenView,
- NestedValueProvider.of(table, new TableRefToJson()),
- getWriteDisposition(),
- getCreateDisposition(),
- tempTablesView,
- getTableDescription()))
- .withSideInputs(tempTablesView, jobIdTokenView));
-
- // Write single partition to final table
- partitions.get(singlePartitionTag)
- .apply("SinglePartitionGroupByKey", GroupByKey.<Long, List<String>>create())
- .apply("SinglePartitionWriteTables", ParDo.of(new WriteTables(
- true,
- getBigQueryServices(),
- jobIdTokenView,
- tempFilePrefix,
- NestedValueProvider.of(table, new TableRefToJson()),
- getJsonSchema(),
- getWriteDisposition(),
- getCreateDisposition(),
- getTableDescription()))
- .withSideInputs(jobIdTokenView));
-
- return PDone.in(input.getPipeline());
- }
-
- private static class WriteBundles extends DoFn<TableRow, KV<String, Long>> {
- private transient TableRowWriter writer = null;
- private final String tempFilePrefix;
-
- WriteBundles(String tempFilePrefix) {
- this.tempFilePrefix = tempFilePrefix;
- }
-
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- if (writer == null) {
- writer = new TableRowWriter(tempFilePrefix);
- writer.open(UUID.randomUUID().toString());
- LOG.debug("Done opening writer {}", writer);
- }
- try {
- writer.write(c.element());
- } catch (Exception e) {
- // Discard write result and close the write.
- try {
- writer.close();
- // The writer does not need to be reset, as this DoFn cannot be reused.
- } catch (Exception closeException) {
- // Do not mask the exception that caused the write to fail.
- e.addSuppressed(closeException);
- }
- throw e;
- }
- }
-
- @FinishBundle
- public void finishBundle(Context c) throws Exception {
- if (writer != null) {
- c.output(writer.close());
- writer = null;
- }
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
-
- builder
- .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
- .withLabel("Temporary File Prefix"));
+ return input.apply(new BatchLoadBigQuery<T>(this));
}
}
@@ -1289,7 +1053,7 @@ public class BigQueryIO {
*
* <p>If the table's project is not specified, use the executing project.
*/
- @Nullable private ValueProvider<TableReference> getTableWithDefaultProject(
+ @Nullable ValueProvider<TableReference> getTableWithDefaultProject(
BigQueryOptions bqOptions) {
ValueProvider<TableReference> table = getTable();
if (table == null) {
@@ -1319,391 +1083,6 @@ public class BigQueryIO {
}
- static class TableRowWriter {
- private static final Coder<TableRow> CODER = TableRowJsonCoder.of();
- private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
- private final String tempFilePrefix;
- private String id;
- private String fileName;
- private WritableByteChannel channel;
- protected String mimeType = MimeTypes.TEXT;
- private CountingOutputStream out;
-
- TableRowWriter(String basename) {
- this.tempFilePrefix = basename;
- }
-
- public final void open(String uId) throws Exception {
- id = uId;
- fileName = tempFilePrefix + id;
- LOG.debug("Opening {}.", fileName);
- channel = IOChannelUtils.create(fileName, mimeType);
- try {
- out = new CountingOutputStream(Channels.newOutputStream(channel));
- LOG.debug("Writing header to {}.", fileName);
- } catch (Exception e) {
- try {
- LOG.error("Writing header to {} failed, closing channel.", fileName);
- channel.close();
- } catch (IOException closeException) {
- LOG.error("Closing channel for {} failed", fileName);
- }
- throw e;
- }
- LOG.debug("Starting write of bundle {} to {}.", this.id, fileName);
- }
-
- public void write(TableRow value) throws Exception {
- CODER.encode(value, out, Context.OUTER);
- out.write(NEWLINE);
- }
-
- public final KV<String, Long> close() throws IOException {
- channel.close();
- return KV.of(fileName, out.getCount());
- }
- }
-
- /**
- * Partitions temporary files based on number of files and file sizes.
- */
- static class WritePartition extends DoFn<String, KV<Long, List<String>>> {
- private final PCollectionView<Iterable<KV<String, Long>>> resultsView;
- private TupleTag<KV<Long, List<String>>> multiPartitionsTag;
- private TupleTag<KV<Long, List<String>>> singlePartitionTag;
-
- public WritePartition(
- PCollectionView<Iterable<KV<String, Long>>> resultsView,
- TupleTag<KV<Long, List<String>>> multiPartitionsTag,
- TupleTag<KV<Long, List<String>>> singlePartitionTag) {
- this.resultsView = resultsView;
- this.multiPartitionsTag = multiPartitionsTag;
- this.singlePartitionTag = singlePartitionTag;
- }
-
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- List<KV<String, Long>> results = Lists.newArrayList(c.sideInput(resultsView));
- if (results.isEmpty()) {
- TableRowWriter writer = new TableRowWriter(c.element());
- writer.open(UUID.randomUUID().toString());
- results.add(writer.close());
- }
-
- long partitionId = 0;
- int currNumFiles = 0;
- long currSizeBytes = 0;
- List<String> currResults = Lists.newArrayList();
- for (int i = 0; i < results.size(); ++i) {
- KV<String, Long> fileResult = results.get(i);
- if (currNumFiles + 1 > Write.MAX_NUM_FILES
- || currSizeBytes + fileResult.getValue() > Write.MAX_SIZE_BYTES) {
- c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
- currResults = Lists.newArrayList();
- currNumFiles = 0;
- currSizeBytes = 0;
- }
- ++currNumFiles;
- currSizeBytes += fileResult.getValue();
- currResults.add(fileResult.getKey());
- }
- if (partitionId == 0) {
- c.sideOutput(singlePartitionTag, KV.of(++partitionId, currResults));
- } else {
- c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
- }
- }
- }
-
- /**
- * Writes partitions to BigQuery tables.
- */
- static class WriteTables extends DoFn<KV<Long, Iterable<List<String>>>, String> {
- private final boolean singlePartition;
- private final BigQueryServices bqServices;
- private final PCollectionView<String> jobIdToken;
- private final String tempFilePrefix;
- private final ValueProvider<String> jsonTableRef;
- private final ValueProvider<String> jsonSchema;
- private final WriteDisposition writeDisposition;
- private final CreateDisposition createDisposition;
- @Nullable private final String tableDescription;
-
- public WriteTables(
- boolean singlePartition,
- BigQueryServices bqServices,
- PCollectionView<String> jobIdToken,
- String tempFilePrefix,
- ValueProvider<String> jsonTableRef,
- ValueProvider<String> jsonSchema,
- WriteDisposition writeDisposition,
- CreateDisposition createDisposition,
- @Nullable String tableDescription) {
- this.singlePartition = singlePartition;
- this.bqServices = bqServices;
- this.jobIdToken = jobIdToken;
- this.tempFilePrefix = tempFilePrefix;
- this.jsonTableRef = jsonTableRef;
- this.jsonSchema = jsonSchema;
- this.writeDisposition = writeDisposition;
- this.createDisposition = createDisposition;
- this.tableDescription = tableDescription;
- }
-
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- List<String> partition = Lists.newArrayList(c.element().getValue()).get(0);
- String jobIdPrefix = String.format(
- c.sideInput(jobIdToken) + "_%05d", c.element().getKey());
- TableReference ref = BigQueryHelpers.fromJsonString(jsonTableRef.get(),
- TableReference.class);
- if (!singlePartition) {
- ref.setTableId(jobIdPrefix);
- }
-
- load(
- bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
- bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
- jobIdPrefix,
- ref,
- BigQueryHelpers.fromJsonString(
- jsonSchema == null ? null : jsonSchema.get(), TableSchema.class),
- partition,
- writeDisposition,
- createDisposition,
- tableDescription);
- c.output(BigQueryHelpers.toJsonString(ref));
-
- removeTemporaryFiles(c.getPipelineOptions(), tempFilePrefix, partition);
- }
-
- private void load(
- JobService jobService,
- DatasetService datasetService,
- String jobIdPrefix,
- TableReference ref,
- @Nullable TableSchema schema,
- List<String> gcsUris,
- WriteDisposition writeDisposition,
- CreateDisposition createDisposition,
- @Nullable String tableDescription) throws InterruptedException, IOException {
- JobConfigurationLoad loadConfig = new JobConfigurationLoad()
- .setDestinationTable(ref)
- .setSchema(schema)
- .setSourceUris(gcsUris)
- .setWriteDisposition(writeDisposition.name())
- .setCreateDisposition(createDisposition.name())
- .setSourceFormat("NEWLINE_DELIMITED_JSON");
-
- String projectId = ref.getProjectId();
- Job lastFailedLoadJob = null;
- for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
- String jobId = jobIdPrefix + "-" + i;
- JobReference jobRef = new JobReference()
- .setProjectId(projectId)
- .setJobId(jobId);
- jobService.startLoadJob(jobRef, loadConfig);
- Job loadJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
- Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
- switch (jobStatus) {
- case SUCCEEDED:
- if (tableDescription != null) {
- datasetService.patchTableDescription(ref, tableDescription);
- }
- return;
- case UNKNOWN:
- throw new RuntimeException(String.format(
- "UNKNOWN status of load job [%s]: %s.", jobId,
- BigQueryHelpers.jobToPrettyString(loadJob)));
- case FAILED:
- lastFailedLoadJob = loadJob;
- continue;
- default:
- throw new IllegalStateException(String.format(
- "Unexpected status [%s] of load job: %s.",
- jobStatus, BigQueryHelpers.jobToPrettyString(loadJob)));
- }
- }
- throw new RuntimeException(String.format(
- "Failed to create load job with id prefix %s, "
- + "reached max retries: %d, last failed load job: %s.",
- jobIdPrefix,
- Write.MAX_RETRY_JOBS,
- BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
- }
-
- static void removeTemporaryFiles(
- PipelineOptions options,
- String tempFilePrefix,
- Collection<String> files)
- throws IOException {
- IOChannelFactory factory = IOChannelUtils.getFactory(tempFilePrefix);
- if (factory instanceof GcsIOChannelFactory) {
- GcsUtil gcsUtil = new GcsUtil.GcsUtilFactory().create(options);
- gcsUtil.remove(files);
- } else if (factory instanceof FileIOChannelFactory) {
- for (String filename : files) {
- LOG.debug("Removing file {}", filename);
- boolean exists = Files.deleteIfExists(Paths.get(filename));
- if (!exists) {
- LOG.debug("{} does not exist.", filename);
- }
- }
- } else {
- throw new IOException("Unrecognized file system.");
- }
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
-
- builder
- .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
- .withLabel("Temporary File Prefix"))
- .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
- .withLabel("Table Reference"))
- .addIfNotNull(DisplayData.item("jsonSchema", jsonSchema)
- .withLabel("Table Schema"))
- .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
- .withLabel("Table Description"));
- }
- }
-
- /**
- * Copies temporary tables to destination table.
- */
- static class WriteRename extends DoFn<String, Void> {
- private final BigQueryServices bqServices;
- private final PCollectionView<String> jobIdToken;
- private final ValueProvider<String> jsonTableRef;
- private final WriteDisposition writeDisposition;
- private final CreateDisposition createDisposition;
- private final PCollectionView<Iterable<String>> tempTablesView;
- @Nullable private final String tableDescription;
-
- public WriteRename(
- BigQueryServices bqServices,
- PCollectionView<String> jobIdToken,
- ValueProvider<String> jsonTableRef,
- WriteDisposition writeDisposition,
- CreateDisposition createDisposition,
- PCollectionView<Iterable<String>> tempTablesView,
- @Nullable String tableDescription) {
- this.bqServices = bqServices;
- this.jobIdToken = jobIdToken;
- this.jsonTableRef = jsonTableRef;
- this.writeDisposition = writeDisposition;
- this.createDisposition = createDisposition;
- this.tempTablesView = tempTablesView;
- this.tableDescription = tableDescription;
- }
-
- @ProcessElement
- public void processElement(ProcessContext c) throws Exception {
- List<String> tempTablesJson = Lists.newArrayList(c.sideInput(tempTablesView));
-
- // Do not copy if no temp tables are provided
- if (tempTablesJson.size() == 0) {
- return;
- }
-
- List<TableReference> tempTables = Lists.newArrayList();
- for (String table : tempTablesJson) {
- tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
- }
- copy(
- bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
- bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
- c.sideInput(jobIdToken),
- BigQueryHelpers.fromJsonString(jsonTableRef.get(), TableReference.class),
- tempTables,
- writeDisposition,
- createDisposition,
- tableDescription);
-
- DatasetService tableService =
- bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class));
- removeTemporaryTables(tableService, tempTables);
- }
-
- private void copy(
- JobService jobService,
- DatasetService datasetService,
- String jobIdPrefix,
- TableReference ref,
- List<TableReference> tempTables,
- WriteDisposition writeDisposition,
- CreateDisposition createDisposition,
- @Nullable String tableDescription) throws InterruptedException, IOException {
- JobConfigurationTableCopy copyConfig = new JobConfigurationTableCopy()
- .setSourceTables(tempTables)
- .setDestinationTable(ref)
- .setWriteDisposition(writeDisposition.name())
- .setCreateDisposition(createDisposition.name());
-
- String projectId = ref.getProjectId();
- Job lastFailedCopyJob = null;
- for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
- String jobId = jobIdPrefix + "-" + i;
- JobReference jobRef = new JobReference()
- .setProjectId(projectId)
- .setJobId(jobId);
- jobService.startCopyJob(jobRef, copyConfig);
- Job copyJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
- Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
- switch (jobStatus) {
- case SUCCEEDED:
- if (tableDescription != null) {
- datasetService.patchTableDescription(ref, tableDescription);
- }
- return;
- case UNKNOWN:
- throw new RuntimeException(String.format(
- "UNKNOWN status of copy job [%s]: %s.", jobId,
- BigQueryHelpers.jobToPrettyString(copyJob)));
- case FAILED:
- lastFailedCopyJob = copyJob;
- continue;
- default:
- throw new IllegalStateException(String.format(
- "Unexpected status [%s] of load job: %s.",
- jobStatus, BigQueryHelpers.jobToPrettyString(copyJob)));
- }
- }
- throw new RuntimeException(String.format(
- "Failed to create copy job with id prefix %s, "
- + "reached max retries: %d, last failed copy job: %s.",
- jobIdPrefix,
- Write.MAX_RETRY_JOBS,
- BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
- }
-
- static void removeTemporaryTables(DatasetService tableService,
- List<TableReference> tempTables) {
- for (TableReference tableRef : tempTables) {
- try {
- LOG.debug("Deleting table {}", BigQueryHelpers.toJsonString(tableRef));
- tableService.deleteTable(tableRef);
- } catch (Exception e) {
- LOG.warn("Failed to delete the table {}", BigQueryHelpers.toJsonString(tableRef), e);
- }
- }
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
-
- builder
- .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
- .withLabel("Table Reference"))
- .add(DisplayData.item("writeDisposition", writeDisposition.toString())
- .withLabel("Write Disposition"))
- .add(DisplayData.item("createDisposition", createDisposition.toString())
- .withLabel("Create Disposition"));
- }
- }
}
private static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
@@ -1753,434 +1132,6 @@ public class BigQueryIO {
static void clearCreatedTables() {
StreamingWriteFn.clearCreatedTables();
}
- /////////////////////////////////////////////////////////////////////////////
-
- /**
- * Implementation of DoFn to perform streaming BigQuery write.
- */
- @VisibleForTesting
- static class StreamingWriteFn
- extends DoFn<KV<ShardedKey<String>, TableRowInfo>, Void> {
- /** TableSchema in JSON. Use String to make the class Serializable. */
- @Nullable private final ValueProvider<String> jsonTableSchema;
-
- @Nullable private final String tableDescription;
-
- private final BigQueryServices bqServices;
-
- /** JsonTableRows to accumulate BigQuery rows in order to batch writes. */
- private transient Map<String, List<TableRow>> tableRows;
-
- private final Write.CreateDisposition createDisposition;
-
- /** The list of unique ids for each BigQuery table row. */
- private transient Map<String, List<String>> uniqueIdsForTableRows;
-
- /** The list of tables created so far, so we don't try the creation
- each time. */
- private static Set<String> createdTables =
- Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
-
- /** Tracks bytes written, exposed as "ByteCount" Metrics Counter. */
- private Counter byteCounter = Metrics.counter(StreamingWriteFn.class, "ByteCount");
-
- /** Constructor. */
- StreamingWriteFn(@Nullable ValueProvider<TableSchema> schema,
- Write.CreateDisposition createDisposition,
- @Nullable String tableDescription, BigQueryServices bqServices) {
- this.jsonTableSchema = schema == null ? null :
- NestedValueProvider.of(schema, new TableSchemaToJsonSchema());
- this.createDisposition = createDisposition;
- this.bqServices = checkNotNull(bqServices, "bqServices");
- this.tableDescription = tableDescription;
- }
-
- /**
- * Clear the cached map of created tables. Used for testing.
- */
- private static void clearCreatedTables() {
- synchronized (createdTables) {
- createdTables.clear();
- }
- }
-
- /** Prepares a target BigQuery table. */
- @StartBundle
- public void startBundle(Context context) {
- tableRows = new HashMap<>();
- uniqueIdsForTableRows = new HashMap<>();
- }
-
- /** Accumulates the input into JsonTableRows and uniqueIdsForTableRows. */
- @ProcessElement
- public void processElement(ProcessContext context) {
- String tableSpec = context.element().getKey().getKey();
- List<TableRow> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
- List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows,
- tableSpec);
-
- rows.add(context.element().getValue().tableRow);
- uniqueIds.add(context.element().getValue().uniqueId);
- }
-
- /** Writes the accumulated rows into BigQuery with streaming API. */
- @FinishBundle
- public void finishBundle(Context context) throws Exception {
- BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class);
-
- for (Map.Entry<String, List<TableRow>> entry : tableRows.entrySet()) {
- TableReference tableReference = getOrCreateTable(options, entry.getKey());
- flushRows(tableReference, entry.getValue(),
- uniqueIdsForTableRows.get(entry.getKey()), options);
- }
- tableRows.clear();
- uniqueIdsForTableRows.clear();
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
-
- builder
- .addIfNotNull(DisplayData.item("schema", jsonTableSchema)
- .withLabel("Table Schema"))
- .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
- .withLabel("Table Description"));
- }
-
- public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
- throws InterruptedException, IOException {
- TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec);
- if (createDisposition != createDisposition.CREATE_NEVER
- && !createdTables.contains(tableSpec)) {
- synchronized (createdTables) {
- // Another thread may have succeeded in creating the table in the meanwhile, so
- // check again. This check isn't needed for correctness, but we add it to prevent
- // every thread from attempting a create and overwhelming our BigQuery quota.
- DatasetService datasetService = bqServices.getDatasetService(options);
- if (!createdTables.contains(tableSpec)) {
- if (datasetService.getTable(tableReference) == null) {
- TableSchema tableSchema = JSON_FACTORY.fromString(
- jsonTableSchema.get(), TableSchema.class);
- datasetService.createTable(
- new Table()
- .setTableReference(tableReference)
- .setSchema(tableSchema)
- .setDescription(tableDescription));
- }
- createdTables.add(tableSpec);
- }
- }
- }
- return tableReference;
- }
-
- /**
- * Writes the accumulated rows into BigQuery with streaming API.
- */
- private void flushRows(TableReference tableReference,
- List<TableRow> tableRows, List<String> uniqueIds, BigQueryOptions options)
- throws InterruptedException {
- if (!tableRows.isEmpty()) {
- try {
- long totalBytes = bqServices.getDatasetService(options).insertAll(
- tableReference, tableRows, uniqueIds);
- byteCounter.inc(totalBytes);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- private static class ShardedKey<K> {
- private final K key;
- private final int shardNumber;
-
- public static <K> ShardedKey<K> of(K key, int shardNumber) {
- return new ShardedKey<>(key, shardNumber);
- }
-
- private ShardedKey(K key, int shardNumber) {
- this.key = key;
- this.shardNumber = shardNumber;
- }
-
- public K getKey() {
- return key;
- }
-
- public int getShardNumber() {
- return shardNumber;
- }
- }
-
- /**
- * A {@link Coder} for {@link ShardedKey}, using a wrapped key {@link Coder}.
- */
- @VisibleForTesting
- static class ShardedKeyCoder<KeyT>
- extends StandardCoder<ShardedKey<KeyT>> {
- public static <KeyT> ShardedKeyCoder<KeyT> of(Coder<KeyT> keyCoder) {
- return new ShardedKeyCoder<>(keyCoder);
- }
-
- @JsonCreator
- public static <KeyT> ShardedKeyCoder<KeyT> of(
- @JsonProperty(PropertyNames.COMPONENT_ENCODINGS)
- List<Coder<KeyT>> components) {
- checkArgument(components.size() == 1, "Expecting 1 component, got %s", components.size());
- return of(components.get(0));
- }
-
- protected ShardedKeyCoder(Coder<KeyT> keyCoder) {
- this.keyCoder = keyCoder;
- this.shardNumberCoder = VarIntCoder.of();
- }
-
- @Override
- public List<? extends Coder<?>> getCoderArguments() {
- return Arrays.asList(keyCoder);
- }
-
- @Override
- public void encode(ShardedKey<KeyT> key, OutputStream outStream, Context context)
- throws IOException {
- keyCoder.encode(key.getKey(), outStream, context.nested());
- shardNumberCoder.encode(key.getShardNumber(), outStream, context);
- }
-
- @Override
- public ShardedKey<KeyT> decode(InputStream inStream, Context context)
- throws IOException {
- return new ShardedKey<>(
- keyCoder.decode(inStream, context.nested()),
- shardNumberCoder.decode(inStream, context));
- }
-
- @Override
- public void verifyDeterministic() throws NonDeterministicException {
- keyCoder.verifyDeterministic();
- }
-
- Coder<KeyT> keyCoder;
- VarIntCoder shardNumberCoder;
- }
-
- @VisibleForTesting
- static class TableRowInfoCoder extends AtomicCoder<TableRowInfo> {
- private static final TableRowInfoCoder INSTANCE = new TableRowInfoCoder();
-
- @JsonCreator
- public static TableRowInfoCoder of() {
- return INSTANCE;
- }
-
- @Override
- public void encode(TableRowInfo value, OutputStream outStream, Context context)
- throws IOException {
- if (value == null) {
- throw new CoderException("cannot encode a null value");
- }
- tableRowCoder.encode(value.tableRow, outStream, context.nested());
- idCoder.encode(value.uniqueId, outStream, context);
- }
-
- @Override
- public TableRowInfo decode(InputStream inStream, Context context)
- throws IOException {
- return new TableRowInfo(
- tableRowCoder.decode(inStream, context.nested()),
- idCoder.decode(inStream, context));
- }
-
- @Override
- public void verifyDeterministic() throws NonDeterministicException {
- throw new NonDeterministicException(this, "TableRows are not deterministic.");
- }
-
- TableRowJsonCoder tableRowCoder = TableRowJsonCoder.of();
- StringUtf8Coder idCoder = StringUtf8Coder.of();
- }
-
- private static class TableRowInfo {
- TableRowInfo(TableRow tableRow, String uniqueId) {
- this.tableRow = tableRow;
- this.uniqueId = uniqueId;
- }
-
- final TableRow tableRow;
- final String uniqueId;
- }
-
- /////////////////////////////////////////////////////////////////////////////
-
- /**
- * Fn that tags each table row with a unique id and destination table.
- * To avoid calling UUID.randomUUID() for each element, which can be costly,
- * a randomUUID is generated only once per bucket of data. The actual unique
- * id is created by concatenating this randomUUID with a sequential number.
- */
- @VisibleForTesting
- static class TagWithUniqueIdsAndTable<T>
- extends DoFn<T, KV<ShardedKey<String>, TableRowInfo>> {
- /** TableSpec to write to. */
- private final ValueProvider<String> tableSpec;
-
- /** User function mapping windowed values to {@link TableReference} in JSON. */
- private final SerializableFunction<ValueInSingleWindow<T>, TableReference> tableRefFunction;
-
- /** User function mapping user type to a TableRow. */
- private final SerializableFunction<T, TableRow> formatFunction;
-
- private transient String randomUUID;
- private transient long sequenceNo = 0L;
-
- TagWithUniqueIdsAndTable(BigQueryOptions options,
- ValueProvider<TableReference> table,
- SerializableFunction<ValueInSingleWindow<T>, TableReference> tableRefFunction,
- SerializableFunction<T, TableRow> formatFunction) {
- checkArgument(table == null ^ tableRefFunction == null,
- "Exactly one of table or tableRefFunction should be set");
- if (table != null) {
- if (table.isAccessible() && Strings.isNullOrEmpty(table.get().getProjectId())) {
- TableReference tableRef = table.get()
- .setProjectId(options.as(BigQueryOptions.class).getProject());
- table = NestedValueProvider.of(
- StaticValueProvider.of(BigQueryHelpers.toJsonString(tableRef)),
- new JsonTableRefToTableRef());
- }
- this.tableSpec = NestedValueProvider.of(table, new TableRefToTableSpec());
- } else {
- tableSpec = null;
- }
- this.tableRefFunction = tableRefFunction;
- this.formatFunction = formatFunction;
- }
-
-
- @StartBundle
- public void startBundle(Context context) {
- randomUUID = UUID.randomUUID().toString();
- }
-
- /** Tag the input with a unique id. */
- @ProcessElement
- public void processElement(ProcessContext context, BoundedWindow window) throws IOException {
- String uniqueId = randomUUID + sequenceNo++;
- ThreadLocalRandom randomGenerator = ThreadLocalRandom.current();
- String tableSpec = tableSpecFromWindowedValue(
- context.getPipelineOptions().as(BigQueryOptions.class),
- ValueInSingleWindow.of(context.element(), context.timestamp(), window, context.pane()));
- // We output on keys 0-50 to ensure that there's enough batching for
- // BigQuery.
- context.output(KV.of(ShardedKey.of(tableSpec, randomGenerator.nextInt(0, 50)),
- new TableRowInfo(formatFunction.apply(context.element()), uniqueId)));
- }
-
- @Override
- public void populateDisplayData(DisplayData.Builder builder) {
- super.populateDisplayData(builder);
-
- builder.addIfNotNull(DisplayData.item("table", tableSpec));
- if (tableRefFunction != null) {
- builder.add(DisplayData.item("tableFn", tableRefFunction.getClass())
- .withLabel("Table Reference Function"));
- }
- }
-
- @VisibleForTesting
- ValueProvider<String> getTableSpec() {
- return tableSpec;
- }
-
- private String tableSpecFromWindowedValue(BigQueryOptions options,
- ValueInSingleWindow<T> value) {
- if (tableSpec != null) {
- return tableSpec.get();
- } else {
- TableReference table = tableRefFunction.apply(value);
- if (table.getProjectId() == null) {
- table.setProjectId(options.getProject());
- }
- return BigQueryHelpers.toTableSpec(table);
- }
- }
- }
-
- /////////////////////////////////////////////////////////////////////////////
-
- /**
- * PTransform that performs streaming BigQuery write. To increase consistency,
- * it leverages BigQuery best effort de-dup mechanism.
- */
- private static class StreamWithDeDup<T> extends PTransform<PCollection<T>, PDone> {
- private final Write<T> write;
-
- /** Constructor. */
- StreamWithDeDup(Write<T> write) {
- this.write = write;
- }
-
- @Override
- protected Coder<Void> getDefaultOutputCoder() {
- return VoidCoder.of();
- }
-
- @Override
- public PDone expand(PCollection<T> input) {
- // A naive implementation would be to simply stream data directly to BigQuery.
- // However, this could occasionally lead to duplicated data, e.g., when
- // a VM that runs this code is restarted and the code is re-run.
-
- // The above risk is mitigated in this implementation by relying on
- // BigQuery built-in best effort de-dup mechanism.
-
- // To use this mechanism, each input TableRow is tagged with a generated
- // unique id, which is then passed to BigQuery and used to ignore duplicates.
-
- PCollection<KV<ShardedKey<String>, TableRowInfo>> tagged =
- input.apply(ParDo.of(new TagWithUniqueIdsAndTable<T>(
- input.getPipeline().getOptions().as(BigQueryOptions.class), write.getTable(),
- write.getTableRefFunction(), write.getFormatFunction())));
-
- // To prevent having the same TableRow processed more than once with regenerated
- // different unique ids, this implementation relies on "checkpointing", which is
- // achieved as a side effect of having StreamingWriteFn immediately follow a GBK,
- // performed by Reshuffle.
- NestedValueProvider<TableSchema, String> schema =
- write.getJsonSchema() == null
- ? null
- : NestedValueProvider.of(write.getJsonSchema(), new JsonSchemaToTableSchema());
- tagged
- .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowInfoCoder.of()))
- .apply(Reshuffle.<ShardedKey<String>, TableRowInfo>of())
- .apply(
- ParDo.of(
- new StreamingWriteFn(
- schema,
- write.getCreateDisposition(),
- write.getTableDescription(),
- write.getBigQueryServices())));
-
- // Note that the implementation to return PDone here breaks the
- // implicit assumption about the job execution order. If a user
- // implements a PTransform that takes PDone returned here as its
- // input, the transform may not necessarily be executed after
- // the BigQueryIO.Write.
-
- return PDone.in(input.getPipeline());
- }
- }
-
- /**
- * Status of a BigQuery job or request.
- */
- enum Status {
- SUCCEEDED,
- FAILED,
- UNKNOWN,
- }
/////////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
index a909957..9153157 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryQuerySource.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -14,9 +32,10 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToJson;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToProjectId;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.options.BigQueryOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
index ff50e6d..746258f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySourceBase.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -18,7 +36,7 @@ import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.coders.TableRowJsonCoder;
import org.apache.beam.sdk.io.AvroSource;
import org.apache.beam.sdk.io.BoundedSource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.options.BigQueryOptions;
import org.apache.beam.sdk.options.PipelineOptions;
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
index aae0faa..cbd5781 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTableSource.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkNotNull;
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
index 612afbe..75f7b93 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/PassThroughThenCleanup.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.beam.sdk.io.gcp.bigquery;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
new file mode 100644
index 0000000..8c968df
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKey.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+/**
+ * A key and a shard number.
+ */
+class ShardedKey<K> {
+ private final K key;
+ private final int shardNumber;
+
+ public static <K> ShardedKey<K> of(K key, int shardNumber) {
+ return new ShardedKey<>(key, shardNumber);
+ }
+
+ ShardedKey(K key, int shardNumber) {
+ this.key = key;
+ this.shardNumber = shardNumber;
+ }
+
+ public K getKey() {
+ return key;
+ }
+
+ public int getShardNumber() {
+ return shardNumber;
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
new file mode 100644
index 0000000..be4e71c
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/ShardedKeyCoder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.StandardCoder;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.beam.sdk.util.PropertyNames;
+
+
+/**
+ * A {@link Coder} for {@link ShardedKey}, using a wrapped key {@link Coder}.
+ */
+@VisibleForTesting
+class ShardedKeyCoder<KeyT>
+ extends StandardCoder<ShardedKey<KeyT>> {
+ public static <KeyT> ShardedKeyCoder<KeyT> of(Coder<KeyT> keyCoder) {
+ return new ShardedKeyCoder<>(keyCoder);
+ }
+
+ @JsonCreator
+ public static <KeyT> ShardedKeyCoder<KeyT> of(
+ @JsonProperty(PropertyNames.COMPONENT_ENCODINGS)
+ List<Coder<KeyT>> components) {
+ checkArgument(components.size() == 1, "Expecting 1 component, got %s", components.size());
+ return of(components.get(0));
+ }
+
+ protected ShardedKeyCoder(Coder<KeyT> keyCoder) {
+ this.keyCoder = keyCoder;
+ this.shardNumberCoder = VarIntCoder.of();
+ }
+
+ @Override
+ public List<? extends Coder<?>> getCoderArguments() {
+ return Arrays.asList(keyCoder);
+ }
+
+ @Override
+ public void encode(ShardedKey<KeyT> key, OutputStream outStream, Context context)
+ throws IOException {
+ keyCoder.encode(key.getKey(), outStream, context.nested());
+ shardNumberCoder.encode(key.getShardNumber(), outStream, context);
+ }
+
+ @Override
+ public ShardedKey<KeyT> decode(InputStream inStream, Context context)
+ throws IOException {
+ return new ShardedKey<>(
+ keyCoder.decode(inStream, context.nested()),
+ shardNumberCoder.decode(inStream, context));
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ keyCoder.verifyDeterministic();
+ }
+
+ Coder<KeyT> keyCoder;
+ VarIntCoder shardNumberCoder;
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java
new file mode 100644
index 0000000..f667295
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamWithDeDup.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableSchema;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.KvCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VoidCoder;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.Reshuffle;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+
+/**
+* PTransform that performs streaming BigQuery write. To increase consistency,
+* it leverages BigQuery best effort de-dup mechanism.
+ */
+class StreamWithDeDup<T> extends PTransform<PCollection<T>, PDone> {
+ private final Write<T> write;
+
+ /** Constructor. */
+ StreamWithDeDup(Write<T> write) {
+ this.write = write;
+ }
+
+ @Override
+ protected Coder<Void> getDefaultOutputCoder() {
+ return VoidCoder.of();
+ }
+
+ @Override
+ public PDone expand(PCollection<T> input) {
+ // A naive implementation would be to simply stream data directly to BigQuery.
+ // However, this could occasionally lead to duplicated data, e.g., when
+ // a VM that runs this code is restarted and the code is re-run.
+
+ // The above risk is mitigated in this implementation by relying on
+ // BigQuery built-in best effort de-dup mechanism.
+
+ // To use this mechanism, each input TableRow is tagged with a generated
+ // unique id, which is then passed to BigQuery and used to ignore duplicates.
+
+ PCollection<KV<ShardedKey<String>, TableRowInfo>> tagged =
+ input.apply(ParDo.of(new TagWithUniqueIdsAndTable<T>(
+ input.getPipeline().getOptions().as(BigQueryOptions.class), write.getTable(),
+ write.getTableRefFunction(), write.getFormatFunction())));
+
+ // To prevent having the same TableRow processed more than once with regenerated
+ // different unique ids, this implementation relies on "checkpointing", which is
+ // achieved as a side effect of having StreamingWriteFn immediately follow a GBK,
+ // performed by Reshuffle.
+ NestedValueProvider<TableSchema, String> schema =
+ write.getJsonSchema() == null
+ ? null
+ : NestedValueProvider.of(write.getJsonSchema(), new JsonSchemaToTableSchema());
+ tagged
+ .setCoder(KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowInfoCoder.of()))
+ .apply(Reshuffle.<ShardedKey<String>, TableRowInfo>of())
+ .apply(
+ ParDo.of(
+ new StreamingWriteFn(
+ schema,
+ write.getCreateDisposition(),
+ write.getTableDescription(),
+ write.getBigQueryServices())));
+
+ // Note that the implementation to return PDone here breaks the
+ // implicit assumption about the job execution order. If a user
+ // implements a PTransform that takes PDone returned here as its
+ // input, the transform may not necessarily be executed after
+ // the BigQueryIO.Write.
+
+ return PDone.in(input.getPipeline());
+ }
+}
[4/6] beam git commit: Refactor BigQueryIO helper methods into a
BigQueryHelpers class. Move helper transforms for BigQueryIO.Read into
individual files. Change private to package in BigQueryHelpers.
Posted by jk...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index 35bb9c3..eca95b9 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -20,8 +20,8 @@ package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.fromJsonString;
-import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.toJsonString;
+import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.fromJsonString;
+import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.toJsonString;
import static org.apache.beam.sdk.transforms.display.DisplayDataMatchers.hasDisplayItem;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasItem;
@@ -101,13 +101,9 @@ import org.apache.beam.sdk.coders.VarLongCoder;
import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.CountingInput;
import org.apache.beam.sdk.io.CountingSource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.BigQueryQuerySource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.BigQueryTableSource;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.JsonSchemaToTableSchema;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.PassThroughThenCleanup;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.PassThroughThenCleanup.CleanupOperation;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TransformingSource;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.TableRowWriter;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
@@ -116,6 +112,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteRename;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteTables;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.io.gcp.bigquery.PassThroughThenCleanup.CleanupOperation;
import org.apache.beam.sdk.options.BigQueryOptions;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -1582,7 +1579,7 @@ public class BigQueryIOTest implements Serializable {
public void testStreamingWriteFnCreateNever() throws Exception {
BigQueryIO.StreamingWriteFn fn = new BigQueryIO.StreamingWriteFn(
null, CreateDisposition.CREATE_NEVER, null, new FakeBigQueryServices());
- assertEquals(BigQueryIO.parseTableSpec("dataset.table"),
+ assertEquals(BigQueryHelpers.parseTableSpec("dataset.table"),
fn.getOrCreateTable(null, "dataset.table"));
}
@@ -1616,7 +1613,7 @@ public class BigQueryIOTest implements Serializable {
@Test
public void testTableParsing() {
- TableReference ref = BigQueryIO
+ TableReference ref = BigQueryHelpers
.parseTableSpec("my-project:data_set.table_name");
Assert.assertEquals("my-project", ref.getProjectId());
Assert.assertEquals("data_set", ref.getDatasetId());
@@ -1625,14 +1622,14 @@ public class BigQueryIOTest implements Serializable {
@Test
public void testTableParsing_validPatterns() {
- BigQueryIO.parseTableSpec("a123-456:foo_bar.d");
- BigQueryIO.parseTableSpec("a12345:b.c");
- BigQueryIO.parseTableSpec("b12345.c");
+ BigQueryHelpers.parseTableSpec("a123-456:foo_bar.d");
+ BigQueryHelpers.parseTableSpec("a12345:b.c");
+ BigQueryHelpers.parseTableSpec("b12345.c");
}
@Test
public void testTableParsing_noProjectId() {
- TableReference ref = BigQueryIO
+ TableReference ref = BigQueryHelpers
.parseTableSpec("data_set.table_name");
Assert.assertEquals(null, ref.getProjectId());
Assert.assertEquals("data_set", ref.getDatasetId());
@@ -1642,25 +1639,25 @@ public class BigQueryIOTest implements Serializable {
@Test
public void testTableParsingError() {
thrown.expect(IllegalArgumentException.class);
- BigQueryIO.parseTableSpec("0123456:foo.bar");
+ BigQueryHelpers.parseTableSpec("0123456:foo.bar");
}
@Test
public void testTableParsingError_2() {
thrown.expect(IllegalArgumentException.class);
- BigQueryIO.parseTableSpec("myproject:.bar");
+ BigQueryHelpers.parseTableSpec("myproject:.bar");
}
@Test
public void testTableParsingError_3() {
thrown.expect(IllegalArgumentException.class);
- BigQueryIO.parseTableSpec(":a.b");
+ BigQueryHelpers.parseTableSpec(":a.b");
}
@Test
public void testTableParsingError_slash() {
thrown.expect(IllegalArgumentException.class);
- BigQueryIO.parseTableSpec("a\\b12345:c.d");
+ BigQueryHelpers.parseTableSpec("a\\b12345:c.d");
}
// Test that BigQuery's special null placeholder objects can be encoded.
@@ -1709,7 +1706,7 @@ public class BigQueryIOTest implements Serializable {
toJsonString(new TableRow().set("name", "c").set("number", "3")));
String jobIdToken = "testJobIdToken";
- TableReference table = BigQueryIO.parseTableSpec("project.data_set.table_name");
+ TableReference table = BigQueryHelpers.parseTableSpec("project.data_set.table_name");
String extractDestinationDir = "mock://tempLocation";
BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
StaticValueProvider.of(jobIdToken), StaticValueProvider.of(table),
@@ -1748,7 +1745,7 @@ public class BigQueryIOTest implements Serializable {
toJsonString(new TableRow().set("name", "c").set("number", "3")));
String jobIdToken = "testJobIdToken";
- TableReference table = BigQueryIO.parseTableSpec("project:data_set.table_name");
+ TableReference table = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
String extractDestinationDir = "mock://tempLocation";
BoundedSource<TableRow> bqSource = BigQueryTableSource.create(
StaticValueProvider.of(jobIdToken), StaticValueProvider.of(table),
@@ -1815,7 +1812,7 @@ public class BigQueryIOTest implements Serializable {
String jobIdToken = "testJobIdToken";
String extractDestinationDir = "mock://tempLocation";
- TableReference destinationTable = BigQueryIO.parseTableSpec("project:data_set.table_name");
+ TableReference destinationTable = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
BoundedSource<TableRow> bqSource = BigQueryQuerySource.create(
StaticValueProvider.of(jobIdToken), StaticValueProvider.of("query"),
StaticValueProvider.of(destinationTable),
@@ -1904,7 +1901,7 @@ public class BigQueryIOTest implements Serializable {
String jobIdToken = "testJobIdToken";
String extractDestinationDir = "mock://tempLocation";
- TableReference destinationTable = BigQueryIO.parseTableSpec("project:data_set.table_name");
+ TableReference destinationTable = BigQueryHelpers.parseTableSpec("project:data_set.table_name");
BoundedSource<TableRow> bqSource = BigQueryQuerySource.create(
StaticValueProvider.of(jobIdToken), StaticValueProvider.of("query"),
StaticValueProvider.of(destinationTable),
@@ -2285,9 +2282,12 @@ public class BigQueryIOTest implements Serializable {
String datasetId = "somedataset";
List<String> tables = Lists.newArrayList("table1", "table2", "table3");
List<TableReference> tableRefs = Lists.newArrayList(
- BigQueryIO.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, tables.get(0))),
- BigQueryIO.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, tables.get(1))),
- BigQueryIO.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, tables.get(2))));
+ BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId,
+ tables.get(0))),
+ BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId,
+ tables.get(1))),
+ BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId,
+ tables.get(2))));
doThrow(new IOException("Unable to delete table"))
.when(mockDatasetService).deleteTable(tableRefs.get(0));
@@ -2371,8 +2371,8 @@ public class BigQueryIOTest implements Serializable {
new BigQueryIO.TagWithUniqueIdsAndTable<TableRow>(
bqOptions, NestedValueProvider.of(
StaticValueProvider.of("data_set.table_name"),
- new BigQueryIO.TableSpecToTableRef()), null, null);
- TableReference table = BigQueryIO.parseTableSpec(tag.getTableSpec().get());
+ new TableSpecToTableRef()), null, null);
+ TableReference table = BigQueryHelpers.parseTableSpec(tag.getTableSpec().get());
assertNotNull(table.getProjectId());
}
http://git-wip-us.apache.org/repos/asf/beam/blob/2cc2e81f/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
index 7b5b226..fa84119 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
@@ -222,7 +222,7 @@ public class BigQueryUtilTest {
List<TableRow> rows = new ArrayList<>();
try (BigQueryTableRowIterator iterator =
BigQueryTableRowIterator.fromTable(
- BigQueryIO.parseTableSpec("project:dataset.table"), mockClient)) {
+ BigQueryHelpers.parseTableSpec("project:dataset.table"), mockClient)) {
iterator.open();
while (iterator.advance()) {
rows.add(iterator.getCurrent());
@@ -261,7 +261,7 @@ public class BigQueryUtilTest {
onTableList(dataList);
try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryIO.parseTableSpec("project:dataset.table"),
+ BigQueryHelpers.parseTableSpec("project:dataset.table"),
mockClient)) {
iterator.open();
Assert.assertTrue(iterator.advance());
@@ -291,7 +291,7 @@ public class BigQueryUtilTest {
onTableList(dataList);
try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryIO.parseTableSpec("project:dataset.table"),
+ BigQueryHelpers.parseTableSpec("project:dataset.table"),
mockClient)) {
iterator.open();
@@ -320,7 +320,7 @@ public class BigQueryUtilTest {
.thenReturn(page2);
try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryIO.parseTableSpec("project:dataset.table"),
+ BigQueryHelpers.parseTableSpec("project:dataset.table"),
mockClient)) {
iterator.open();
@@ -350,7 +350,7 @@ public class BigQueryUtilTest {
.thenThrow(new IOException("No such table"));
try (BigQueryTableRowIterator iterator = BigQueryTableRowIterator.fromTable(
- BigQueryIO.parseTableSpec("project:dataset.table"),
+ BigQueryHelpers.parseTableSpec("project:dataset.table"),
mockClient)) {
try {
iterator.open(); // throws.
@@ -387,7 +387,7 @@ public class BigQueryUtilTest {
errorsIndices.add(new ArrayList<Long>());
onInsertAll(errorsIndices);
- TableReference ref = BigQueryIO
+ TableReference ref = BigQueryHelpers
.parseTableSpec("project:dataset.table");
DatasetServiceImpl datasetService = new DatasetServiceImpl(mockClient, options, 5);
[2/6] beam git commit: Refactor BigQueryIO.Write helper transforms
into separate files.
Posted by jk...@apache.org.
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
new file mode 100644
index 0000000..1d93fa3
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteFn.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSchemaToJsonSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.SystemDoFnInternal;
+import org.apache.beam.sdk.values.KV;
+
+/**
+ * Implementation of DoFn to perform streaming BigQuery write.
+ */
+@SystemDoFnInternal
+@VisibleForTesting
+class StreamingWriteFn
+ extends DoFn<KV<ShardedKey<String>, TableRowInfo>, Void> {
+ /** TableSchema in JSON. Use String to make the class Serializable. */
+ @Nullable
+ private final ValueProvider<String> jsonTableSchema;
+
+ @Nullable private final String tableDescription;
+
+ private final BigQueryServices bqServices;
+
+ /** JsonTableRows to accumulate BigQuery rows in order to batch writes. */
+ private transient Map<String, List<TableRow>> tableRows;
+
+ private final Write.CreateDisposition createDisposition;
+
+ /** The list of unique ids for each BigQuery table row. */
+ private transient Map<String, List<String>> uniqueIdsForTableRows;
+
+ /** The list of tables created so far, so we don't try the creation
+ each time. */
+ private static Set<String> createdTables =
+ Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+
+ /** Tracks bytes written, exposed as "ByteCount" Counter. */
+ private Counter byteCounter = Metrics.counter(StreamingWriteFn.class, "ByteCount");
+
+ /** Constructor. */
+ StreamingWriteFn(@Nullable ValueProvider<TableSchema> schema,
+ Write.CreateDisposition createDisposition,
+ @Nullable String tableDescription, BigQueryServices bqServices) {
+ this.jsonTableSchema = schema == null ? null :
+ NestedValueProvider.of(schema, new TableSchemaToJsonSchema());
+ this.createDisposition = createDisposition;
+ this.bqServices = checkNotNull(bqServices, "bqServices");
+ this.tableDescription = tableDescription;
+ }
+
+ /**
+ * Clear the cached map of created tables. Used for testing.
+ */
+ static void clearCreatedTables() {
+ synchronized (createdTables) {
+ createdTables.clear();
+ }
+ }
+
+ /** Prepares a target BigQuery table. */
+ @StartBundle
+ public void startBundle(Context context) {
+ tableRows = new HashMap<>();
+ uniqueIdsForTableRows = new HashMap<>();
+ }
+
+ /** Accumulates the input into JsonTableRows and uniqueIdsForTableRows. */
+ @ProcessElement
+ public void processElement(ProcessContext context) {
+ String tableSpec = context.element().getKey().getKey();
+ List<TableRow> rows = BigQueryHelpers.getOrCreateMapListValue(tableRows, tableSpec);
+ List<String> uniqueIds = BigQueryHelpers.getOrCreateMapListValue(uniqueIdsForTableRows,
+ tableSpec);
+
+ rows.add(context.element().getValue().tableRow);
+ uniqueIds.add(context.element().getValue().uniqueId);
+ }
+
+ /** Writes the accumulated rows into BigQuery with streaming API. */
+ @FinishBundle
+ public void finishBundle(Context context) throws Exception {
+ BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class);
+
+ for (Map.Entry<String, List<TableRow>> entry : tableRows.entrySet()) {
+ TableReference tableReference = getOrCreateTable(options, entry.getKey());
+ flushRows(tableReference, entry.getValue(),
+ uniqueIdsForTableRows.get(entry.getKey()), options);
+ }
+ tableRows.clear();
+ uniqueIdsForTableRows.clear();
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder
+ .addIfNotNull(DisplayData.item("schema", jsonTableSchema)
+ .withLabel("Table Schema"))
+ .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
+ .withLabel("Table Description"));
+ }
+
+ public TableReference getOrCreateTable(BigQueryOptions options, String tableSpec)
+ throws InterruptedException, IOException {
+ TableReference tableReference = BigQueryHelpers.parseTableSpec(tableSpec);
+ if (createDisposition != createDisposition.CREATE_NEVER
+ && !createdTables.contains(tableSpec)) {
+ synchronized (createdTables) {
+ // Another thread may have succeeded in creating the table in the meanwhile, so
+ // check again. This check isn't needed for correctness, but we add it to prevent
+ // every thread from attempting a create and overwhelming our BigQuery quota.
+ DatasetService datasetService = bqServices.getDatasetService(options);
+ if (!createdTables.contains(tableSpec)) {
+ if (datasetService.getTable(tableReference) == null) {
+ TableSchema tableSchema = BigQueryIO.JSON_FACTORY.fromString(
+ jsonTableSchema.get(), TableSchema.class);
+ datasetService.createTable(
+ new Table()
+ .setTableReference(tableReference)
+ .setSchema(tableSchema)
+ .setDescription(tableDescription));
+ }
+ createdTables.add(tableSpec);
+ }
+ }
+ }
+ return tableReference;
+ }
+
+ /**
+ * Writes the accumulated rows into BigQuery with streaming API.
+ */
+ private void flushRows(TableReference tableReference,
+ List<TableRow> tableRows, List<String> uniqueIds, BigQueryOptions options)
+ throws InterruptedException {
+ if (!tableRows.isEmpty()) {
+ try {
+ long totalBytes = bqServices.getDatasetService(options).insertAll(
+ tableReference, tableRows, uniqueIds);
+ byteCounter.inc(totalBytes);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java
new file mode 100644
index 0000000..4a59ab6
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfo.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableRow;
+
+/**
+ * Encapsulates a {@link TableRow} along with a unique insertion id.
+ */
+class TableRowInfo {
+ TableRowInfo(TableRow tableRow, String uniqueId) {
+ this.tableRow = tableRow;
+ this.uniqueId = uniqueId;
+ }
+
+ final TableRow tableRow;
+ final String uniqueId;
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
new file mode 100644
index 0000000..5e8fa29
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowInfoCoder.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import org.apache.beam.sdk.coders.AtomicCoder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.TableRowJsonCoder;
+
+/**
+ * Defines a coder for {@link TableRowInfo} objects.
+ */
+@VisibleForTesting
+class TableRowInfoCoder extends AtomicCoder<TableRowInfo> {
+ private static final TableRowInfoCoder INSTANCE = new TableRowInfoCoder();
+
+ @JsonCreator
+ public static TableRowInfoCoder of() {
+ return INSTANCE;
+ }
+
+ @Override
+ public void encode(TableRowInfo value, OutputStream outStream, Context context)
+ throws IOException {
+ if (value == null) {
+ throw new CoderException("cannot encode a null value");
+ }
+ tableRowCoder.encode(value.tableRow, outStream, context.nested());
+ idCoder.encode(value.uniqueId, outStream, context);
+ }
+
+ @Override
+ public TableRowInfo decode(InputStream inStream, Context context)
+ throws IOException {
+ return new TableRowInfo(
+ tableRowCoder.decode(inStream, context.nested()),
+ idCoder.decode(inStream, context));
+ }
+
+ @Override
+ public void verifyDeterministic() throws NonDeterministicException {
+ throw new NonDeterministicException(this, "TableRows are not deterministic.");
+ }
+
+ TableRowJsonCoder tableRowCoder = TableRowJsonCoder.of();
+ StringUtf8Coder idCoder = StringUtf8Coder.of();
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
new file mode 100644
index 0000000..014c498
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TableRowWriter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.io.CountingOutputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.nio.charset.StandardCharsets;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.Coder.Context;
+import org.apache.beam.sdk.coders.TableRowJsonCoder;
+import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.MimeTypes;
+import org.apache.beam.sdk.values.KV;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Writes {@TableRow} objects out to a file. Used when doing batch load jobs into BigQuery.
+ */
+class TableRowWriter {
+ private static final Logger LOG = LoggerFactory.getLogger(BigQueryIO.class);
+
+ private static final Coder<TableRow> CODER = TableRowJsonCoder.of();
+ private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
+ private final String tempFilePrefix;
+ private String id;
+ private String fileName;
+ private WritableByteChannel channel;
+ protected String mimeType = MimeTypes.TEXT;
+ private CountingOutputStream out;
+
+ TableRowWriter(String basename) {
+ this.tempFilePrefix = basename;
+ }
+
+ public final void open(String uId) throws Exception {
+ id = uId;
+ fileName = tempFilePrefix + id;
+ LOG.debug("Opening {}.", fileName);
+ channel = IOChannelUtils.create(fileName, mimeType);
+ try {
+ out = new CountingOutputStream(Channels.newOutputStream(channel));
+ LOG.debug("Writing header to {}.", fileName);
+ } catch (Exception e) {
+ try {
+ LOG.error("Writing header to {} failed, closing channel.", fileName);
+ channel.close();
+ } catch (IOException closeException) {
+ LOG.error("Closing channel for {} failed", fileName);
+ }
+ throw e;
+ }
+ LOG.debug("Starting write of bundle {} to {}.", this.id, fileName);
+ }
+
+ public void write(TableRow value) throws Exception {
+ CODER.encode(value, out, Context.OUTER);
+ out.write(NEWLINE);
+ }
+
+ public final KV<String, Long> close() throws IOException {
+ channel.close();
+ return KV.of(fileName, out.getCount());
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java
new file mode 100644
index 0000000..a6608e4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TagWithUniqueIdsAndTable.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonTableRefToTableRef;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableRefToTableSpec;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
+
+/**
+ * Fn that tags each table row with a unique id and destination table.
+ * To avoid calling UUID.randomUUID() for each element, which can be costly,
+ * a randomUUID is generated only once per bucket of data. The actual unique
+ * id is created by concatenating this randomUUID with a sequential number.
+ */
+@VisibleForTesting
+class TagWithUniqueIdsAndTable<T>
+ extends DoFn<T, KV<ShardedKey<String>, TableRowInfo>> {
+ /** TableSpec to write to. */
+ private final ValueProvider<String> tableSpec;
+
+ /** User function mapping windowed values to {@link TableReference} in JSON. */
+ private final SerializableFunction<ValueInSingleWindow<T>, TableReference> tableRefFunction;
+
+ /** User function mapping user type to a TableRow. */
+ private final SerializableFunction<T, TableRow> formatFunction;
+
+ private transient String randomUUID;
+ private transient long sequenceNo = 0L;
+
+ TagWithUniqueIdsAndTable(BigQueryOptions options,
+ ValueProvider<TableReference> table,
+ SerializableFunction<ValueInSingleWindow<T>, TableReference>
+ tableRefFunction,
+ SerializableFunction<T, TableRow> formatFunction) {
+ checkArgument(table == null ^ tableRefFunction == null,
+ "Exactly one of table or tableRefFunction should be set");
+ if (table != null) {
+ if (table.isAccessible() && Strings.isNullOrEmpty(table.get().getProjectId())) {
+ TableReference tableRef = table.get()
+ .setProjectId(options.as(BigQueryOptions.class).getProject());
+ table = NestedValueProvider.of(
+ StaticValueProvider.of(BigQueryHelpers.toJsonString(tableRef)),
+ new JsonTableRefToTableRef());
+ }
+ this.tableSpec = NestedValueProvider.of(table, new TableRefToTableSpec());
+ } else {
+ tableSpec = null;
+ }
+ this.tableRefFunction = tableRefFunction;
+ this.formatFunction = formatFunction;
+ }
+
+
+ @StartBundle
+ public void startBundle(Context context) {
+ randomUUID = UUID.randomUUID().toString();
+ }
+
+ /** Tag the input with a unique id. */
+ @ProcessElement
+ public void processElement(ProcessContext context, BoundedWindow window) throws IOException {
+ String uniqueId = randomUUID + sequenceNo++;
+ ThreadLocalRandom randomGenerator = ThreadLocalRandom.current();
+ String tableSpec = tableSpecFromWindowedValue(
+ context.getPipelineOptions().as(BigQueryOptions.class),
+ ValueInSingleWindow.of(context.element(), context.timestamp(), window, context.pane()));
+ // We output on keys 0-50 to ensure that there's enough batching for
+ // BigQuery.
+ context.output(KV.of(ShardedKey.of(tableSpec, randomGenerator.nextInt(0, 50)),
+ new TableRowInfo(formatFunction.apply(context.element()), uniqueId)));
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder.addIfNotNull(DisplayData.item("table", tableSpec));
+ if (tableRefFunction != null) {
+ builder.add(DisplayData.item("tableFn", tableRefFunction.getClass())
+ .withLabel("Table Reference Function"));
+ }
+ }
+
+ @VisibleForTesting
+ ValueProvider<String> getTableSpec() {
+ return tableSpec;
+ }
+
+ private String tableSpecFromWindowedValue(BigQueryOptions options,
+ ValueInSingleWindow<T> value) {
+ if (tableSpec != null) {
+ return tableSpec.get();
+ } else {
+ TableReference table = tableRefFunction.apply(value);
+ if (table.getProjectId() == null) {
+ table.setProjectId(options.getProject());
+ }
+ return BigQueryHelpers.toTableSpec(table);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
index a86adfb..f7d8252 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/TransformingSource.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.beam.sdk.io.gcp.bigquery;
import static com.google.common.base.Preconditions.checkNotNull;
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java
new file mode 100644
index 0000000..6219226
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteBundles.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.TableRow;
+import java.util.UUID;
+
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.KV;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Writes each bundle of {@link TableRow} elements out to a separate file using
+ * {@link TableRowWriter}.
+ */
+class WriteBundles extends DoFn<TableRow, KV<String, Long>> {
+ private static final Logger LOG = LoggerFactory.getLogger(WriteBundles.class);
+
+ private transient TableRowWriter writer = null;
+ private final String tempFilePrefix;
+
+ WriteBundles(String tempFilePrefix) {
+ this.tempFilePrefix = tempFilePrefix;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ if (writer == null) {
+ writer = new TableRowWriter(tempFilePrefix);
+ writer.open(UUID.randomUUID().toString());
+ LOG.debug("Done opening writer {}", writer);
+ }
+ try {
+ writer.write(c.element());
+ } catch (Exception e) {
+ // Discard write result and close the write.
+ try {
+ writer.close();
+ // The writer does not need to be reset, as this DoFn cannot be reused.
+ } catch (Exception closeException) {
+ // Do not mask the exception that caused the write to fail.
+ e.addSuppressed(closeException);
+ }
+ throw e;
+ }
+ }
+
+ @FinishBundle
+ public void finishBundle(Context c) throws Exception {
+ if (writer != null) {
+ c.output(writer.close());
+ writer = null;
+ }
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder
+ .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
+ .withLabel("Temporary File Prefix"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
new file mode 100644
index 0000000..f4bf198
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WritePartition.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.common.collect.Lists;
+import java.util.List;
+import java.util.UUID;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.apache.beam.sdk.values.TupleTag;
+
+/**
+ * Partitions temporary files based on number of files and file sizes.
+ */
+class WritePartition extends DoFn<String, KV<Long, List<String>>> {
+ private final PCollectionView<Iterable<KV<String, Long>>> resultsView;
+ private TupleTag<KV<Long, List<String>>> multiPartitionsTag;
+ private TupleTag<KV<Long, List<String>>> singlePartitionTag;
+
+ public WritePartition(
+ PCollectionView<Iterable<KV<String, Long>>> resultsView,
+ TupleTag<KV<Long, List<String>>> multiPartitionsTag,
+ TupleTag<KV<Long, List<String>>> singlePartitionTag) {
+ this.resultsView = resultsView;
+ this.multiPartitionsTag = multiPartitionsTag;
+ this.singlePartitionTag = singlePartitionTag;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ List<KV<String, Long>> results = Lists.newArrayList(c.sideInput(resultsView));
+ if (results.isEmpty()) {
+ TableRowWriter writer = new TableRowWriter(c.element());
+ writer.open(UUID.randomUUID().toString());
+ results.add(writer.close());
+ }
+
+ long partitionId = 0;
+ int currNumFiles = 0;
+ long currSizeBytes = 0;
+ List<String> currResults = Lists.newArrayList();
+ for (int i = 0; i < results.size(); ++i) {
+ KV<String, Long> fileResult = results.get(i);
+ if (currNumFiles + 1 > Write.MAX_NUM_FILES
+ || currSizeBytes + fileResult.getValue() > Write.MAX_SIZE_BYTES) {
+ c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
+ currResults = Lists.newArrayList();
+ currNumFiles = 0;
+ currSizeBytes = 0;
+ }
+ ++currNumFiles;
+ currSizeBytes += fileResult.getValue();
+ currResults.add(fileResult.getKey());
+ }
+ if (partitionId == 0) {
+ c.sideOutput(singlePartitionTag, KV.of(++partitionId, currResults));
+ } else {
+ c.sideOutput(multiPartitionsTag, KV.of(++partitionId, currResults));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
new file mode 100644
index 0000000..8cb9439
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteRename.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.util.List;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Copies temporary tables to destination table.
+ */
+class WriteRename extends DoFn<String, Void> {
+ private static final Logger LOG = LoggerFactory.getLogger(WriteRename.class);
+
+ private final BigQueryServices bqServices;
+ private final PCollectionView<String> jobIdToken;
+ private final ValueProvider<String> jsonTableRef;
+ private final WriteDisposition writeDisposition;
+ private final CreateDisposition createDisposition;
+ private final PCollectionView<Iterable<String>> tempTablesView;
+ @Nullable
+ private final String tableDescription;
+
+ public WriteRename(
+ BigQueryServices bqServices,
+ PCollectionView<String> jobIdToken,
+ ValueProvider<String> jsonTableRef,
+ WriteDisposition writeDisposition,
+ CreateDisposition createDisposition,
+ PCollectionView<Iterable<String>> tempTablesView,
+ @Nullable String tableDescription) {
+ this.bqServices = bqServices;
+ this.jobIdToken = jobIdToken;
+ this.jsonTableRef = jsonTableRef;
+ this.writeDisposition = writeDisposition;
+ this.createDisposition = createDisposition;
+ this.tempTablesView = tempTablesView;
+ this.tableDescription = tableDescription;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ List<String> tempTablesJson = Lists.newArrayList(c.sideInput(tempTablesView));
+
+ // Do not copy if no temp tables are provided
+ if (tempTablesJson.size() == 0) {
+ return;
+ }
+
+ List<TableReference> tempTables = Lists.newArrayList();
+ for (String table : tempTablesJson) {
+ tempTables.add(BigQueryHelpers.fromJsonString(table, TableReference.class));
+ }
+ copy(
+ bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ c.sideInput(jobIdToken),
+ BigQueryHelpers.fromJsonString(jsonTableRef.get(), TableReference.class),
+ tempTables,
+ writeDisposition,
+ createDisposition,
+ tableDescription);
+
+ DatasetService tableService =
+ bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class));
+ removeTemporaryTables(tableService, tempTables);
+ }
+
+ private void copy(
+ JobService jobService,
+ DatasetService datasetService,
+ String jobIdPrefix,
+ TableReference ref,
+ List<TableReference> tempTables,
+ WriteDisposition writeDisposition,
+ CreateDisposition createDisposition,
+ @Nullable String tableDescription) throws InterruptedException, IOException {
+ JobConfigurationTableCopy copyConfig = new JobConfigurationTableCopy()
+ .setSourceTables(tempTables)
+ .setDestinationTable(ref)
+ .setWriteDisposition(writeDisposition.name())
+ .setCreateDisposition(createDisposition.name());
+
+ String projectId = ref.getProjectId();
+ Job lastFailedCopyJob = null;
+ for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
+ String jobId = jobIdPrefix + "-" + i;
+ JobReference jobRef = new JobReference()
+ .setProjectId(projectId)
+ .setJobId(jobId);
+ jobService.startCopyJob(jobRef, copyConfig);
+ Job copyJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
+ Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
+ switch (jobStatus) {
+ case SUCCEEDED:
+ if (tableDescription != null) {
+ datasetService.patchTableDescription(ref, tableDescription);
+ }
+ return;
+ case UNKNOWN:
+ throw new RuntimeException(String.format(
+ "UNKNOWN status of copy job [%s]: %s.", jobId,
+ BigQueryHelpers.jobToPrettyString(copyJob)));
+ case FAILED:
+ lastFailedCopyJob = copyJob;
+ continue;
+ default:
+ throw new IllegalStateException(String.format(
+ "Unexpected status [%s] of load job: %s.",
+ jobStatus, BigQueryHelpers.jobToPrettyString(copyJob)));
+ }
+ }
+ throw new RuntimeException(String.format(
+ "Failed to create copy job with id prefix %s, "
+ + "reached max retries: %d, last failed copy job: %s.",
+ jobIdPrefix,
+ Write.MAX_RETRY_JOBS,
+ BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
+ }
+
+ static void removeTemporaryTables(DatasetService tableService,
+ List<TableReference> tempTables) {
+ for (TableReference tableRef : tempTables) {
+ try {
+ LOG.debug("Deleting table {}", BigQueryHelpers.toJsonString(tableRef));
+ tableService.deleteTable(tableRef);
+ } catch (Exception e) {
+ LOG.warn("Failed to delete the table {}", BigQueryHelpers.toJsonString(tableRef), e);
+ }
+ }
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder
+ .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
+ .withLabel("Table Reference"))
+ .add(DisplayData.item("writeDisposition", writeDisposition.toString())
+ .withLabel("Write Disposition"))
+ .add(DisplayData.item("createDisposition", createDisposition.toString())
+ .withLabel("Create Disposition"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
new file mode 100644
index 0000000..29680ad
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/WriteTables.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.api.services.bigquery.model.TableSchema;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.List;
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
+import org.apache.beam.sdk.options.BigQueryOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.options.ValueProvider;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.FileIOChannelFactory;
+import org.apache.beam.sdk.util.GcsIOChannelFactory;
+import org.apache.beam.sdk.util.GcsUtil;
+import org.apache.beam.sdk.util.GcsUtil.GcsUtilFactory;
+import org.apache.beam.sdk.util.IOChannelFactory;
+import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollectionView;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Writes partitions to BigQuery tables.
+ */
+class WriteTables extends DoFn<KV<Long, Iterable<List<String>>>, String> {
+ private static final Logger LOG = LoggerFactory.getLogger(WriteTables.class);
+
+ private final boolean singlePartition;
+ private final BigQueryServices bqServices;
+ private final PCollectionView<String> jobIdToken;
+ private final String tempFilePrefix;
+ private final ValueProvider<String> jsonTableRef;
+ private final ValueProvider<String> jsonSchema;
+ private final WriteDisposition writeDisposition;
+ private final CreateDisposition createDisposition;
+ @Nullable
+ private final String tableDescription;
+
+ public WriteTables(
+ boolean singlePartition,
+ BigQueryServices bqServices,
+ PCollectionView<String> jobIdToken,
+ String tempFilePrefix,
+ ValueProvider<String> jsonTableRef,
+ ValueProvider<String> jsonSchema,
+ WriteDisposition writeDisposition,
+ CreateDisposition createDisposition,
+ @Nullable String tableDescription) {
+ this.singlePartition = singlePartition;
+ this.bqServices = bqServices;
+ this.jobIdToken = jobIdToken;
+ this.tempFilePrefix = tempFilePrefix;
+ this.jsonTableRef = jsonTableRef;
+ this.jsonSchema = jsonSchema;
+ this.writeDisposition = writeDisposition;
+ this.createDisposition = createDisposition;
+ this.tableDescription = tableDescription;
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) throws Exception {
+ List<String> partition = Lists.newArrayList(c.element().getValue()).get(0);
+ String jobIdPrefix = String.format(
+ c.sideInput(jobIdToken) + "_%05d", c.element().getKey());
+ TableReference ref = BigQueryHelpers.fromJsonString(jsonTableRef.get(),
+ TableReference.class);
+ if (!singlePartition) {
+ ref.setTableId(jobIdPrefix);
+ }
+
+ load(
+ bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)),
+ jobIdPrefix,
+ ref,
+ BigQueryHelpers.fromJsonString(
+ jsonSchema == null ? null : jsonSchema.get(), TableSchema.class),
+ partition,
+ writeDisposition,
+ createDisposition,
+ tableDescription);
+ c.output(BigQueryHelpers.toJsonString(ref));
+
+ removeTemporaryFiles(c.getPipelineOptions(), tempFilePrefix, partition);
+ }
+
+ private void load(
+ JobService jobService,
+ DatasetService datasetService,
+ String jobIdPrefix,
+ TableReference ref,
+ @Nullable TableSchema schema,
+ List<String> gcsUris,
+ WriteDisposition writeDisposition,
+ CreateDisposition createDisposition,
+ @Nullable String tableDescription) throws InterruptedException, IOException {
+ JobConfigurationLoad loadConfig = new JobConfigurationLoad()
+ .setDestinationTable(ref)
+ .setSchema(schema)
+ .setSourceUris(gcsUris)
+ .setWriteDisposition(writeDisposition.name())
+ .setCreateDisposition(createDisposition.name())
+ .setSourceFormat("NEWLINE_DELIMITED_JSON");
+
+ String projectId = ref.getProjectId();
+ Job lastFailedLoadJob = null;
+ for (int i = 0; i < Write.MAX_RETRY_JOBS; ++i) {
+ String jobId = jobIdPrefix + "-" + i;
+ JobReference jobRef = new JobReference()
+ .setProjectId(projectId)
+ .setJobId(jobId);
+ jobService.startLoadJob(jobRef, loadConfig);
+ Job loadJob = jobService.pollJob(jobRef, Write.LOAD_JOB_POLL_MAX_RETRIES);
+ Status jobStatus = BigQueryHelpers.parseStatus(loadJob);
+ switch (jobStatus) {
+ case SUCCEEDED:
+ if (tableDescription != null) {
+ datasetService.patchTableDescription(ref, tableDescription);
+ }
+ return;
+ case UNKNOWN:
+ throw new RuntimeException(String.format(
+ "UNKNOWN status of load job [%s]: %s.", jobId,
+ BigQueryHelpers.jobToPrettyString(loadJob)));
+ case FAILED:
+ lastFailedLoadJob = loadJob;
+ continue;
+ default:
+ throw new IllegalStateException(String.format(
+ "Unexpected status [%s] of load job: %s.",
+ jobStatus, BigQueryHelpers.jobToPrettyString(loadJob)));
+ }
+ }
+ throw new RuntimeException(String.format(
+ "Failed to create load job with id prefix %s, "
+ + "reached max retries: %d, last failed load job: %s.",
+ jobIdPrefix,
+ Write.MAX_RETRY_JOBS,
+ BigQueryHelpers.jobToPrettyString(lastFailedLoadJob)));
+ }
+
+ static void removeTemporaryFiles(
+ PipelineOptions options,
+ String tempFilePrefix,
+ Collection<String> files)
+ throws IOException {
+ IOChannelFactory factory = IOChannelUtils.getFactory(tempFilePrefix);
+ if (factory instanceof GcsIOChannelFactory) {
+ GcsUtil gcsUtil = new GcsUtilFactory().create(options);
+ gcsUtil.remove(files);
+ } else if (factory instanceof FileIOChannelFactory) {
+ for (String filename : files) {
+ LOG.debug("Removing file {}", filename);
+ boolean exists = Files.deleteIfExists(Paths.get(filename));
+ if (!exists) {
+ LOG.debug("{} does not exist.", filename);
+ }
+ }
+ } else {
+ throw new IOException("Unrecognized file system.");
+ }
+ }
+
+ @Override
+ public void populateDisplayData(DisplayData.Builder builder) {
+ super.populateDisplayData(builder);
+
+ builder
+ .addIfNotNull(DisplayData.item("tempFilePrefix", tempFilePrefix)
+ .withLabel("Temporary File Prefix"))
+ .addIfNotNull(DisplayData.item("jsonTableRef", jsonTableRef)
+ .withLabel("Table Reference"))
+ .addIfNotNull(DisplayData.item("jsonSchema", jsonSchema)
+ .withLabel("Table Schema"))
+ .addIfNotNull(DisplayData.item("tableDescription", tableDescription)
+ .withLabel("Table Description"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d6ef0104/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
index eca95b9..e7db8a2 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTest.java
@@ -102,14 +102,10 @@ import org.apache.beam.sdk.io.BoundedSource;
import org.apache.beam.sdk.io.CountingInput;
import org.apache.beam.sdk.io.CountingSource;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.Status;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.TableSpecToTableRef;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Status;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.TableRowWriter;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WritePartition;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteRename;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteTables;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.JobService;
import org.apache.beam.sdk.io.gcp.bigquery.PassThroughThenCleanup.CleanupOperation;
@@ -654,7 +650,9 @@ public class BigQueryIOTest implements Serializable {
@Rule public final transient TestPipeline p = TestPipeline.create();
@Rule public transient ExpectedException thrown = ExpectedException.none();
- @Rule public transient ExpectedLogs logged = ExpectedLogs.none(BigQueryIO.class);
+ @Rule public transient ExpectedLogs loggedBigQueryIO = ExpectedLogs.none(BigQueryIO.class);
+ @Rule public transient ExpectedLogs loggedWriteRename = ExpectedLogs.none(WriteRename.class);
+ @Rule public transient ExpectedLogs loggedWriteTables = ExpectedLogs.none(WriteTables.class);
@Rule public transient TemporaryFolder testFolder = new TemporaryFolder();
@Mock(extraInterfaces = Serializable.class)
public transient BigQueryServices.JobService mockJobService;
@@ -1577,7 +1575,7 @@ public class BigQueryIOTest implements Serializable {
@Test
public void testStreamingWriteFnCreateNever() throws Exception {
- BigQueryIO.StreamingWriteFn fn = new BigQueryIO.StreamingWriteFn(
+ StreamingWriteFn fn = new StreamingWriteFn(
null, CreateDisposition.CREATE_NEVER, null, new FakeBigQueryServices());
assertEquals(BigQueryHelpers.parseTableSpec("dataset.table"),
fn.getOrCreateTable(null, "dataset.table"));
@@ -2230,9 +2228,9 @@ public class BigQueryIOTest implements Serializable {
testNumFiles(tempDir, 0);
for (String fileName : fileNames) {
- logged.verifyDebug("Removing file " + fileName);
+ loggedWriteTables.verifyDebug("Removing file " + fileName);
}
- logged.verifyDebug(fileNames.get(numFiles) + " does not exist.");
+ loggedWriteTables.verifyDebug(fileNames.get(numFiles) + " does not exist.");
}
@Test
@@ -2297,11 +2295,14 @@ public class BigQueryIOTest implements Serializable {
WriteRename.removeTemporaryTables(mockDatasetService, tableRefs);
for (TableReference ref : tableRefs) {
- logged.verifyDebug("Deleting table " + toJsonString(ref));
+ loggedWriteRename.verifyDebug("Deleting table " + toJsonString(ref));
}
- logged.verifyWarn("Failed to delete the table " + toJsonString(tableRefs.get(0)));
- logged.verifyNotLogged("Failed to delete the table " + toJsonString(tableRefs.get(1)));
- logged.verifyNotLogged("Failed to delete the table " + toJsonString(tableRefs.get(2)));
+ loggedWriteRename.verifyWarn("Failed to delete the table "
+ + toJsonString(tableRefs.get(0)));
+ loggedWriteRename.verifyNotLogged("Failed to delete the table "
+ + toJsonString(tableRefs.get(1)));
+ loggedWriteRename.verifyNotLogged("Failed to delete the table "
+ + toJsonString(tableRefs.get(2)));
}
/** Test options. **/
@@ -2367,8 +2368,8 @@ public class BigQueryIOTest implements Serializable {
public void testTagWithUniqueIdsAndTableProjectNotNullWithNvp() {
BigQueryOptions bqOptions = PipelineOptionsFactory.as(BigQueryOptions.class);
bqOptions.setProject("project");
- BigQueryIO.TagWithUniqueIdsAndTable<TableRow> tag =
- new BigQueryIO.TagWithUniqueIdsAndTable<TableRow>(
+ TagWithUniqueIdsAndTable<TableRow> tag =
+ new TagWithUniqueIdsAndTable<TableRow>(
bqOptions, NestedValueProvider.of(
StaticValueProvider.of("data_set.table_name"),
new TableSpecToTableRef()), null, null);
@@ -2423,12 +2424,12 @@ public class BigQueryIOTest implements Serializable {
@Test
public void testShardedKeyCoderIsSerializableWithWellKnownCoderType() {
- CoderProperties.coderSerializable(BigQueryIO.ShardedKeyCoder.of(GlobalWindow.Coder.INSTANCE));
+ CoderProperties.coderSerializable(ShardedKeyCoder.of(GlobalWindow.Coder.INSTANCE));
}
@Test
public void testTableRowInfoCoderSerializable() {
- CoderProperties.coderSerializable(BigQueryIO.TableRowInfoCoder.of());
+ CoderProperties.coderSerializable(TableRowInfoCoder.of());
}
@Test
@@ -2436,8 +2437,8 @@ public class BigQueryIOTest implements Serializable {
CoderProperties.coderSerializable(
WindowedValue.getFullCoder(
KvCoder.of(
- BigQueryIO.ShardedKeyCoder.of(StringUtf8Coder.of()),
- BigQueryIO.TableRowInfoCoder.of()),
+ ShardedKeyCoder.of(StringUtf8Coder.of()),
+ TableRowInfoCoder.of()),
IntervalWindow.getCoder()));
}
}
[6/6] beam git commit: This closes #2271
Posted by jk...@apache.org.
This closes #2271
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1339dd70
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1339dd70
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1339dd70
Branch: refs/heads/master
Commit: 1339dd70623d4a1737e10d5cdc22416ffdaf6d06
Parents: 434eadb 2381dee
Author: Eugene Kirpichov <ki...@google.com>
Authored: Tue Mar 28 08:46:35 2017 -0700
Committer: Eugene Kirpichov <ki...@google.com>
Committed: Tue Mar 28 08:46:35 2017 -0700
----------------------------------------------------------------------
.../sdk/io/gcp/bigquery/BatchLoadBigQuery.java | 182 ++
.../sdk/io/gcp/bigquery/BigQueryHelpers.java | 339 ++++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 1915 +-----------------
.../io/gcp/bigquery/BigQueryQuerySource.java | 205 ++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 2 +-
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 196 ++
.../io/gcp/bigquery/BigQueryTableSource.java | 104 +
.../io/gcp/bigquery/PassThroughThenCleanup.java | 84 +
.../beam/sdk/io/gcp/bigquery/ShardedKey.java | 44 +
.../sdk/io/gcp/bigquery/ShardedKeyCoder.java | 87 +
.../sdk/io/gcp/bigquery/StreamWithDeDup.java | 98 +
.../sdk/io/gcp/bigquery/StreamingWriteFn.java | 186 ++
.../beam/sdk/io/gcp/bigquery/TableRowInfo.java | 34 +
.../sdk/io/gcp/bigquery/TableRowInfoCoder.java | 68 +
.../sdk/io/gcp/bigquery/TableRowWriter.java | 84 +
.../gcp/bigquery/TagWithUniqueIdsAndTable.java | 135 ++
.../sdk/io/gcp/bigquery/TransformingSource.java | 136 ++
.../beam/sdk/io/gcp/bigquery/WriteBundles.java | 82 +
.../sdk/io/gcp/bigquery/WritePartition.java | 79 +
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 180 ++
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 213 ++
.../sdk/io/gcp/bigquery/BigQueryIOTest.java | 93 +-
.../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 12 +-
23 files changed, 2627 insertions(+), 1931 deletions(-)
----------------------------------------------------------------------