You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/04/12 00:16:24 UTC
[1/2] incubator-beam git commit: [BEAM-50] Implement BigQueryIO.Write
as a custom sink.
Repository: incubator-beam
Updated Branches:
refs/heads/master d2b659b7c -> 936d028f0
[BEAM-50] Implement BigQueryIO.Write as a custom sink.
Create BigQueryServices interface and added BigQueryIO pipeline tests.
Removed BigQueryIO.Write evaluator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a97d79e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a97d79e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a97d79e5
Branch: refs/heads/master
Commit: a97d79e50c9483945dde42b49d53e93fea315ed3
Parents: d2b659b
Author: Pei He <pe...@gmail.com>
Authored: Mon Mar 14 21:26:19 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Apr 11 15:15:58 2016 -0700
----------------------------------------------------------------------
.../cloud/dataflow/sdk/io/BigQueryIO.java | 378 ++++++++++++++-----
.../dataflow/sdk/util/BigQueryServices.java | 61 +++
.../dataflow/sdk/util/BigQueryServicesImpl.java | 196 ++++++++++
.../cloud/dataflow/sdk/io/BigQueryIOTest.java | 192 +++++++++-
.../sdk/util/BigQueryServicesImplTest.java | 262 +++++++++++++
5 files changed, 975 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a97d79e5/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java
index ae0ce4d..c37d6c7 100644
--- a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java
+++ b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/io/BigQueryIO.java
@@ -17,14 +17,19 @@
*/
package com.google.cloud.dataflow.sdk.io;
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
import com.google.api.client.json.JsonFactory;
import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
import com.google.api.services.bigquery.model.QueryRequest;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.cloud.dataflow.sdk.coders.AtomicCoder;
import com.google.cloud.dataflow.sdk.coders.Coder;
+import com.google.cloud.dataflow.sdk.coders.Coder.Context;
import com.google.cloud.dataflow.sdk.coders.CoderException;
import com.google.cloud.dataflow.sdk.coders.KvCoder;
import com.google.cloud.dataflow.sdk.coders.StandardCoder;
@@ -36,6 +41,7 @@ import com.google.cloud.dataflow.sdk.io.BigQueryIO.Write.CreateDisposition;
import com.google.cloud.dataflow.sdk.io.BigQueryIO.Write.WriteDisposition;
import com.google.cloud.dataflow.sdk.options.BigQueryOptions;
import com.google.cloud.dataflow.sdk.options.GcpOptions;
+import com.google.cloud.dataflow.sdk.options.PipelineOptions;
import com.google.cloud.dataflow.sdk.runners.DirectPipelineRunner;
import com.google.cloud.dataflow.sdk.runners.PipelineRunner;
import com.google.cloud.dataflow.sdk.transforms.Aggregator;
@@ -45,22 +51,28 @@ import com.google.cloud.dataflow.sdk.transforms.ParDo;
import com.google.cloud.dataflow.sdk.transforms.SerializableFunction;
import com.google.cloud.dataflow.sdk.transforms.Sum;
import com.google.cloud.dataflow.sdk.transforms.windowing.BoundedWindow;
+import com.google.cloud.dataflow.sdk.util.BigQueryServices;
+import com.google.cloud.dataflow.sdk.util.BigQueryServices.LoadService;
+import com.google.cloud.dataflow.sdk.util.BigQueryServicesImpl;
import com.google.cloud.dataflow.sdk.util.BigQueryTableInserter;
import com.google.cloud.dataflow.sdk.util.BigQueryTableRowIterator;
+import com.google.cloud.dataflow.sdk.util.MimeTypes;
import com.google.cloud.dataflow.sdk.util.PropertyNames;
import com.google.cloud.dataflow.sdk.util.Reshuffle;
import com.google.cloud.dataflow.sdk.util.SystemDoFnInternal;
import com.google.cloud.dataflow.sdk.util.Transport;
-import com.google.cloud.dataflow.sdk.util.WindowedValue;
import com.google.cloud.dataflow.sdk.util.WindowingStrategy;
+import com.google.cloud.dataflow.sdk.util.gcsfs.GcsPath;
import com.google.cloud.dataflow.sdk.values.KV;
import com.google.cloud.dataflow.sdk.values.PCollection;
import com.google.cloud.dataflow.sdk.values.PCollection.IsBounded;
import com.google.cloud.dataflow.sdk.values.PDone;
import com.google.cloud.dataflow.sdk.values.PInput;
import com.google.cloud.hadoop.util.ApiErrorExtractor;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
-import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -71,6 +83,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -725,12 +740,12 @@ public class BigQueryIO {
* {@link PCollection} of {@link TableRow TableRows} to a BigQuery table.
*/
public static class Bound extends PTransform<PCollection<TableRow>, PDone> {
- final TableReference table;
+ @Nullable final String jsonTableRef;
- final SerializableFunction<BoundedWindow, TableReference> tableRefFunction;
+ @Nullable final SerializableFunction<BoundedWindow, TableReference> tableRefFunction;
// Table schema. The schema is required only if the table does not exist.
- final TableSchema schema;
+ @Nullable final String jsonSchema;
// Options for creating the table. Valid values are CREATE_IF_NEEDED and
// CREATE_NEVER.
@@ -743,6 +758,9 @@ public class BigQueryIO {
// An option to indicate if table validation is desired. Default is true.
final boolean validate;
+ // A fake or mock BigQueryServices for tests.
+ @Nullable private BigQueryServices testBigQueryServices;
+
private static class TranslateTableSpecFunction implements
SerializableFunction<BoundedWindow, TableReference> {
private SerializableFunction<BoundedWindow, String> tableSpecFunction;
@@ -765,20 +783,22 @@ public class BigQueryIO {
@Deprecated
public Bound() {
this(null, null, null, null, CreateDisposition.CREATE_IF_NEEDED,
- WriteDisposition.WRITE_EMPTY, true);
+ WriteDisposition.WRITE_EMPTY, true, null);
}
- private Bound(String name, TableReference ref,
- SerializableFunction<BoundedWindow, TableReference> tableRefFunction, TableSchema schema,
- CreateDisposition createDisposition, WriteDisposition writeDisposition,
- boolean validate) {
+ private Bound(String name, @Nullable String jsonTableRef,
+ @Nullable SerializableFunction<BoundedWindow, TableReference> tableRefFunction,
+ @Nullable String jsonSchema,
+ CreateDisposition createDisposition, WriteDisposition writeDisposition, boolean validate,
+ @Nullable BigQueryServices testBigQueryServices) {
super(name);
- this.table = ref;
+ this.jsonTableRef = jsonTableRef;
this.tableRefFunction = tableRefFunction;
- this.schema = schema;
- this.createDisposition = createDisposition;
- this.writeDisposition = writeDisposition;
+ this.jsonSchema = jsonSchema;
+ this.createDisposition = checkNotNull(createDisposition, "createDisposition");
+ this.writeDisposition = checkNotNull(writeDisposition, "writeDisposition");
this.validate = validate;
+ this.testBigQueryServices = testBigQueryServices;
}
/**
@@ -787,8 +807,8 @@ public class BigQueryIO {
* <p>Does not modify this object.
*/
public Bound named(String name) {
- return new Bound(name, table, tableRefFunction, schema, createDisposition,
- writeDisposition, validate);
+ return new Bound(name, jsonTableRef, tableRefFunction, jsonSchema, createDisposition,
+ writeDisposition, validate, testBigQueryServices);
}
/**
@@ -807,8 +827,8 @@ public class BigQueryIO {
* <p>Does not modify this object.
*/
public Bound to(TableReference table) {
- return new Bound(name, table, tableRefFunction, schema, createDisposition,
- writeDisposition, validate);
+ return new Bound(name, toJsonString(table), tableRefFunction, jsonSchema, createDisposition,
+ writeDisposition, validate, testBigQueryServices);
}
/**
@@ -836,8 +856,8 @@ public class BigQueryIO {
*/
public Bound toTableReference(
SerializableFunction<BoundedWindow, TableReference> tableRefFunction) {
- return new Bound(name, table, tableRefFunction, schema, createDisposition,
- writeDisposition, validate);
+ return new Bound(name, jsonTableRef, tableRefFunction, jsonSchema, createDisposition,
+ writeDisposition, validate, testBigQueryServices);
}
/**
@@ -847,8 +867,8 @@ public class BigQueryIO {
* <p>Does not modify this object.
*/
public Bound withSchema(TableSchema schema) {
- return new Bound(name, table, tableRefFunction, schema, createDisposition,
- writeDisposition, validate);
+ return new Bound(name, jsonTableRef, tableRefFunction, toJsonString(schema),
+ createDisposition, writeDisposition, validate, testBigQueryServices);
}
/**
@@ -857,8 +877,8 @@ public class BigQueryIO {
* <p>Does not modify this object.
*/
public Bound withCreateDisposition(CreateDisposition createDisposition) {
- return new Bound(name, table, tableRefFunction, schema, createDisposition,
- writeDisposition, validate);
+ return new Bound(name, jsonTableRef, tableRefFunction, jsonSchema, createDisposition,
+ writeDisposition, validate, testBigQueryServices);
}
/**
@@ -867,8 +887,8 @@ public class BigQueryIO {
* <p>Does not modify this object.
*/
public Bound withWriteDisposition(WriteDisposition writeDisposition) {
- return new Bound(name, table, tableRefFunction, schema, createDisposition,
- writeDisposition, validate);
+ return new Bound(name, jsonTableRef, tableRefFunction, jsonSchema, createDisposition,
+ writeDisposition, validate, testBigQueryServices);
}
/**
@@ -877,8 +897,14 @@ public class BigQueryIO {
* <p>Does not modify this object.
*/
public Bound withoutValidation() {
- return new Bound(name, table, tableRefFunction, schema, createDisposition,
- writeDisposition, false);
+ return new Bound(name, jsonTableRef, tableRefFunction, jsonSchema, createDisposition,
+ writeDisposition, false, testBigQueryServices);
+ }
+
+ @VisibleForTesting
+ Bound withTestServices(BigQueryServices testServices) {
+ return new Bound(name, jsonTableRef, tableRefFunction, jsonSchema, createDisposition,
+ writeDisposition, validate, testServices);
}
private static void verifyTableEmpty(
@@ -907,6 +933,7 @@ public class BigQueryIO {
public PDone apply(PCollection<TableRow> input) {
BigQueryOptions options = input.getPipeline().getOptions().as(BigQueryOptions.class);
+ TableReference table = getTable();
if (table == null && tableRefFunction == null) {
throw new IllegalStateException(
"must set the table reference of a BigQueryIO.Write transform");
@@ -917,7 +944,7 @@ public class BigQueryIO {
+ "transform");
}
- if (createDisposition == CreateDisposition.CREATE_IF_NEEDED && schema == null) {
+ if (createDisposition == CreateDisposition.CREATE_IF_NEEDED && jsonSchema == null) {
throw new IllegalArgumentException("CreateDisposition is CREATE_IF_NEEDED, "
+ "however no schema was provided.");
}
@@ -959,10 +986,34 @@ public class BigQueryIO {
+ "supported for unbounded PCollections or when using tablespec functions.");
}
- return input.apply(new StreamWithDeDup(table, tableRefFunction, schema));
+ return input.apply(new StreamWithDeDup(table, tableRefFunction, getSchema()));
}
- return PDone.in(input.getPipeline());
+ String tempLocation = options.getTempLocation();
+ checkArgument(!Strings.isNullOrEmpty(tempLocation),
+ "BigQueryIO.Write needs a GCS temp location to store temp files.");
+ if (testBigQueryServices == null) {
+ try {
+ GcsPath.fromUri(tempLocation);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(String.format(
+ "BigQuery temp location expected a valid 'gs://' path, but was given '%s'",
+ tempLocation), e);
+ }
+ }
+ String jobIdToken = UUID.randomUUID().toString();
+ String tempFilePrefix = tempLocation + "/BigQuerySinkTemp/" + jobIdToken;
+ BigQueryServices bqServices = getBigQueryServices();
+ return input.apply("Write", com.google.cloud.dataflow.sdk.io.Write.to(
+ new BigQuerySink(
+ jobIdToken,
+ jsonTableRef,
+ jsonSchema,
+ getWriteDisposition(),
+ getCreateDisposition(),
+ tempFilePrefix,
+ input.getCoder(),
+ bqServices)));
}
@Override
@@ -970,17 +1021,6 @@ public class BigQueryIO {
return VoidCoder.of();
}
- static {
- DirectPipelineRunner.registerDefaultTransformEvaluator(
- Bound.class, new DirectPipelineRunner.TransformEvaluator<Bound>() {
- @Override
- public void evaluate(
- Bound transform, DirectPipelineRunner.EvaluationContext context) {
- evaluateWriteHelper(transform, context);
- }
- });
- }
-
/** Returns the create disposition. */
public CreateDisposition getCreateDisposition() {
return createDisposition;
@@ -993,24 +1033,191 @@ public class BigQueryIO {
/** Returns the table schema. */
public TableSchema getSchema() {
- return schema;
+ return fromJsonString(jsonSchema, TableSchema.class);
}
/** Returns the table reference, or {@code null} if a . */
public TableReference getTable() {
- return table;
+ return fromJsonString(jsonTableRef, TableReference.class);
}
/** Returns {@code true} if table validation is enabled. */
public boolean getValidate() {
return validate;
}
+
+ private BigQueryServices getBigQueryServices() {
+ if (testBigQueryServices != null) {
+ return testBigQueryServices;
+ } else {
+ return new BigQueryServicesImpl();
+ }
+ }
}
/** Disallow construction of utility class. */
private Write() {}
}
+ /**
+ * {@link BigQuerySink} is implemented as a {@link FileBasedSink}.
+ *
+ * <p>It uses BigQuery load job to import files into BigQuery.
+ */
+ static class BigQuerySink extends FileBasedSink<TableRow> {
+ private final String jobIdToken;
+ @Nullable private final String jsonTable;
+ @Nullable private final String jsonSchema;
+ private final WriteDisposition writeDisposition;
+ private final CreateDisposition createDisposition;
+ private final Coder<TableRow> coder;
+ private final BigQueryServices bqServices;
+
+ public BigQuerySink(
+ String jobIdToken,
+ @Nullable String jsonTable,
+ @Nullable String jsonSchema,
+ WriteDisposition writeDisposition,
+ CreateDisposition createDisposition,
+ String tempFile,
+ Coder<TableRow> coder,
+ BigQueryServices bqServices) {
+ super(tempFile, ".json");
+ this.jobIdToken = checkNotNull(jobIdToken, "jobIdToken");
+ this.jsonTable = jsonTable;
+ this.jsonSchema = jsonSchema;
+ this.writeDisposition = checkNotNull(writeDisposition, "writeDisposition");
+ this.createDisposition = checkNotNull(createDisposition, "createDisposition");
+ this.coder = checkNotNull(coder, "coder");
+ this.bqServices = checkNotNull(bqServices, "bqServices");
+ }
+
+ @Override
+ public FileBasedSink.FileBasedWriteOperation<TableRow> createWriteOperation(
+ PipelineOptions options) {
+ return new BigQueryWriteOperation(this);
+ }
+
+ private static class BigQueryWriteOperation extends FileBasedWriteOperation<TableRow> {
+ // The maximum number of retry load jobs.
+ private static final int MAX_RETRY_LOAD_JOBS = 3;
+
+ private final BigQuerySink bigQuerySink;
+
+ private BigQueryWriteOperation(BigQuerySink sink) {
+ super(checkNotNull(sink, "sink"));
+ this.bigQuerySink = sink;
+ }
+
+ @Override
+ public FileBasedWriter<TableRow> createWriter(PipelineOptions options) throws Exception {
+ return new TableRowWriter(this, bigQuerySink.coder);
+ }
+
+ @Override
+ public void finalize(Iterable<FileResult> writerResults, PipelineOptions options)
+ throws IOException, InterruptedException {
+ try {
+ BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+ List<String> tempFiles = Lists.newArrayList();
+ for (FileResult result : writerResults) {
+ tempFiles.add(result.getFilename());
+ }
+ if (!tempFiles.isEmpty()) {
+ load(
+ bigQuerySink.bqServices.getLoadService(bqOptions),
+ bigQuerySink.jobIdToken,
+ fromJsonString(bigQuerySink.jsonTable, TableReference.class),
+ tempFiles,
+ fromJsonString(bigQuerySink.jsonSchema, TableSchema.class),
+ bigQuerySink.writeDisposition,
+ bigQuerySink.createDisposition);
+ }
+ } finally {
+ removeTemporaryFiles(options);
+ }
+ }
+
+ /**
+ * Import files into BigQuery with load jobs.
+ *
+ * <p>Returns if files are successfully loaded into BigQuery.
+ * Throws a RuntimeException if:
+ * 1. The status of one load job is UNKNOWN. This is to avoid duplicating data.
+ * 2. It exceeds {@code MAX_RETRY_LOAD_JOBS}.
+ *
+ * <p>If a load job failed, it will try another load job with a different job id.
+ */
+ private void load(
+ LoadService loadService,
+ String jobIdPrefix,
+ TableReference ref,
+ List<String> gcsUris,
+ @Nullable TableSchema schema,
+ WriteDisposition writeDisposition,
+ CreateDisposition createDisposition) throws InterruptedException, IOException {
+ JobConfigurationLoad loadConfig = new JobConfigurationLoad();
+ loadConfig.setSourceUris(gcsUris);
+ loadConfig.setDestinationTable(ref);
+ loadConfig.setSchema(schema);
+ loadConfig.setWriteDisposition(writeDisposition.name());
+ loadConfig.setCreateDisposition(createDisposition.name());
+ loadConfig.setSourceFormat("NEWLINE_DELIMITED_JSON");
+
+ boolean retrying = false;
+ String projectId = ref.getProjectId();
+ for (int i = 0; i < MAX_RETRY_LOAD_JOBS; ++i) {
+ String jobId = jobIdPrefix + "-" + i;
+ if (retrying) {
+ LOG.info("Previous load jobs failed, retrying.");
+ }
+ LOG.info("Starting BigQuery load job: {}", jobId);
+ loadService.startLoadJob(jobId, loadConfig);
+ BigQueryServices.Status jobStatus = loadService.pollJobStatus(projectId, jobId);
+ switch (jobStatus) {
+ case SUCCEEDED:
+ return;
+ case UNKNOWN:
+ throw new RuntimeException("Failed to poll the load job status.");
+ case FAILED:
+ LOG.info("BigQuery load job failed: {}", jobId);
+ retrying = true;
+ continue;
+ default:
+ throw new IllegalStateException("Unexpected job status: " + jobStatus);
+ }
+ }
+ throw new RuntimeException(
+ "Failed to create the load job, reached max retries: " + MAX_RETRY_LOAD_JOBS);
+ }
+ }
+
+ private static class TableRowWriter extends FileBasedWriter<TableRow> {
+ private static final byte[] NEWLINE = "\n".getBytes(StandardCharsets.UTF_8);
+ private final Coder<TableRow> coder;
+ private OutputStream out;
+
+ public TableRowWriter(
+ FileBasedWriteOperation<TableRow> writeOperation, Coder<TableRow> coder) {
+ super(writeOperation);
+ this.mimeType = MimeTypes.TEXT;
+ this.coder = coder;
+ }
+
+ @Override
+ protected void prepareWrite(WritableByteChannel channel) throws Exception {
+ out = Channels.newOutputStream(channel);
+ }
+
+ @Override
+ public void write(TableRow value) throws Exception {
+ // Use Context.OUTER to encode and NEWLINE as the delimeter.
+ coder.encode(value, out, Context.OUTER);
+ out.write(NEWLINE);
+ }
+ }
+ }
+
private static void verifyDatasetPresence(BigQueryOptions options, TableReference table) {
try {
Bigquery client = Transport.newBigQueryClient(options).build();
@@ -1080,11 +1287,7 @@ public class BigQueryIO {
/** Constructor. */
StreamingWriteFn(TableSchema schema) {
- try {
- jsonTableSchema = JSON_FACTORY.toString(schema);
- } catch (IOException e) {
- throw new RuntimeException("Cannot initialize BigQuery streaming writer.", e);
- }
+ jsonTableSchema = toJsonString(schema);
}
/** Prepares a target BigQuery table. */
@@ -1190,8 +1393,7 @@ public class BigQueryIO {
public static <KeyT> ShardedKeyCoder<KeyT> of(
@JsonProperty(PropertyNames.COMPONENT_ENCODINGS)
List<Coder<KeyT>> components) {
- Preconditions.checkArgument(components.size() == 1,
- "Expecting 1 component, got " + components.size());
+ checkArgument(components.size() == 1, "Expecting 1 component, got %s", components.size());
return of(components.get(0));
}
@@ -1296,7 +1498,7 @@ public class BigQueryIO {
TagWithUniqueIdsAndTable(BigQueryOptions options, TableReference table,
SerializableFunction<BoundedWindow, TableReference> tableRefFunction) {
- Preconditions.checkArgument(table == null ^ tableRefFunction == null,
+ checkArgument(table == null ^ tableRefFunction == null,
"Exactly one of table or tableRefFunction should be set");
if (table != null) {
if (table.getProjectId() == null) {
@@ -1401,6 +1603,32 @@ public class BigQueryIO {
}
}
+ private static String toJsonString(Object item) {
+ if (item == null) {
+ return null;
+ }
+ try {
+ return JSON_FACTORY.toString(item);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format("Cannot serialize %s to a JSON string.", item.getClass().getSimpleName()),
+ e);
+ }
+ }
+
+ private static <T> T fromJsonString(String json, Class<T> clazz) {
+ if (json == null) {
+ return null;
+ }
+ try {
+ return JSON_FACTORY.fromString(json, clazz);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ String.format("Cannot deserialize %s from a JSON string: %s.", clazz, json),
+ e);
+ }
+ }
+
/////////////////////////////////////////////////////////////////////////////
/** Disallow construction of utility class. */
@@ -1454,50 +1682,4 @@ public class BigQueryIO {
}
return value;
}
-
- /**
- * Direct mode write evaluator.
- *
- * <p>This writes the entire table in a single BigQuery request.
- * The table will be created if necessary.
- */
- private static void evaluateWriteHelper(
- Write.Bound transform, DirectPipelineRunner.EvaluationContext context) {
- BigQueryOptions options = context.getPipelineOptions();
- Bigquery client = Transport.newBigQueryClient(options).build();
- BigQueryTableInserter inserter = new BigQueryTableInserter(client);
-
- try {
- Map<TableReference, List<TableRow>> tableRows = new HashMap<>();
- for (WindowedValue<TableRow> windowedValue : context.getPCollectionWindowedValues(
- context.getInput(transform))) {
- for (BoundedWindow window : windowedValue.getWindows()) {
- TableReference ref;
- if (transform.tableRefFunction != null) {
- ref = transform.tableRefFunction.apply(window);
- } else {
- ref = transform.table;
- }
- if (ref.getProjectId() == null) {
- ref.setProjectId(options.getProject());
- }
-
- List<TableRow> rows = getOrCreateMapListValue(tableRows, ref);
- rows.add(windowedValue.getValue());
- }
- }
-
- for (TableReference ref : tableRows.keySet()) {
- LOG.info("Writing to BigQuery table {}", toTableSpec(ref));
- // {@link BigQueryTableInserter#getOrCreateTable} validates {@link CreateDisposition}
- // and {@link WriteDisposition}.
- // For each {@link TableReference}, it can only be called before rows are written.
- inserter.getOrCreateTable(
- ref, transform.writeDisposition, transform.createDisposition, transform.schema);
- inserter.insertAll(ref, tableRows.get(ref));
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a97d79e5/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServices.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServices.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServices.java
new file mode 100644
index 0000000..b39b2e8
--- /dev/null
+++ b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServices.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.dataflow.sdk.util;
+
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
+import com.google.cloud.dataflow.sdk.options.BigQueryOptions;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+/**
+ * An interface for real, mock, or fake implementations of Cloud BigQuery services.
+ */
+public interface BigQueryServices extends Serializable {
+
+ /**
+ * Status of a BigQuery job or request.
+ */
+ enum Status {
+ SUCCEEDED,
+ FAILED,
+ UNKNOWN,
+ }
+
+ /**
+ * Returns a real, mock, or fake {@link LoadService}.
+ */
+ public LoadService getLoadService(BigQueryOptions bqOptions);
+
+ /**
+ * An interface for the Cloud BigQuery load service.
+ */
+ public interface LoadService {
+ /**
+ * Start a BigQuery load job.
+ */
+ public void startLoadJob(String jobId, JobConfigurationLoad loadConfig)
+ throws InterruptedException, IOException;
+
+ /**
+ * Poll the status of a BigQuery load job.
+ */
+ public Status pollJobStatus(String projectId, String jobId)
+ throws InterruptedException, IOException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a97d79e5/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java
new file mode 100644
index 0000000..f0c8864
--- /dev/null
+++ b/sdks/java/core/src/main/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImpl.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.dataflow.sdk.util;
+
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.util.BackOff;
+import com.google.api.client.util.BackOffUtils;
+import com.google.api.client.util.Sleeper;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfiguration;
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
+import com.google.api.services.bigquery.model.JobReference;
+import com.google.api.services.bigquery.model.JobStatus;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.dataflow.sdk.options.BigQueryOptions;
+import com.google.cloud.hadoop.util.ApiErrorExtractor;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An implementation of {@link BigQueryServices} that actually communicates with the cloud BigQuery
+ * service.
+ */
+public class BigQueryServicesImpl implements BigQueryServices {
+
+ // The maximum number of attempts to execute a load job RPC.
+ private static final int MAX_LOAD_JOB_RPC_ATTEMPTS = 10;
+
+ // The initial backoff for executing a load job RPC.
+ private static final long INITIAL_LOAD_JOB_RPC_BACKOFF_MILLIS = TimeUnit.SECONDS.toMillis(1);
+ // The maximum number of retries to poll the status of a load job.
+ // It sets to {@code Integer.MAX_VALUE} to block until the BigQuery load job finishes.
+ private static final int MAX_LOAD_JOB_POLL_RETRIES = Integer.MAX_VALUE;
+
+ // The initial backoff for polling the status of a load job.
+ private static final long INITIAL_LOAD_JOB_POLL_BACKOFF_MILLIS = TimeUnit.SECONDS.toMillis(60);
+
+ @Override
+ public LoadService getLoadService(BigQueryOptions options) {
+ return new LoadServiceImpl(options);
+ }
+
+ @VisibleForTesting
+ static class LoadServiceImpl implements BigQueryServices.LoadService {
+ private static final Logger LOG = LoggerFactory.getLogger(LoadServiceImpl.class);
+
+ private final ApiErrorExtractor errorExtractor;
+ private final Bigquery client;
+
+ @VisibleForTesting
+ LoadServiceImpl(Bigquery client) {
+ this.errorExtractor = new ApiErrorExtractor();
+ this.client = client;
+ }
+
+ private LoadServiceImpl(BigQueryOptions options) {
+ this.errorExtractor = new ApiErrorExtractor();
+ this.client = Transport.newBigQueryClient(options).build();
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Retries the RPC for at most {@code MAX_LOAD_JOB_RPC_ATTEMPTS} times until it succeeds.
+ *
+ * @throws IOException if it exceeds max RPC retries.
+ */
+ @Override
+ public void startLoadJob(
+ String jobId,
+ JobConfigurationLoad loadConfig) throws InterruptedException, IOException {
+ BackOff backoff = new AttemptBoundedExponentialBackOff(
+ MAX_LOAD_JOB_RPC_ATTEMPTS, INITIAL_LOAD_JOB_RPC_BACKOFF_MILLIS);
+ startLoadJob(jobId, loadConfig, Sleeper.DEFAULT, backoff);
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Retries the poll request for at most {@code MAX_LOAD_JOB_POLL_RETRIES} times
+ * until the job is DONE.
+ */
+ @Override
+ public Status pollJobStatus(String projectId, String jobId) throws InterruptedException {
+ BackOff backoff = new AttemptBoundedExponentialBackOff(
+ MAX_LOAD_JOB_POLL_RETRIES, INITIAL_LOAD_JOB_POLL_BACKOFF_MILLIS);
+ return pollJobStatus(projectId, jobId, Sleeper.DEFAULT, backoff);
+ }
+
+ @VisibleForTesting
+ void startLoadJob(
+ String jobId,
+ JobConfigurationLoad loadConfig,
+ Sleeper sleeper,
+ BackOff backoff)
+ throws InterruptedException, IOException {
+ TableReference ref = loadConfig.getDestinationTable();
+ String projectId = ref.getProjectId();
+
+ Job job = new Job();
+ JobReference jobRef = new JobReference();
+ jobRef.setProjectId(projectId);
+ jobRef.setJobId(jobId);
+ job.setJobReference(jobRef);
+ JobConfiguration config = new JobConfiguration();
+ config.setLoad(loadConfig);
+ job.setConfiguration(config);
+
+ Exception lastException = null;
+ do {
+ try {
+ client.jobs().insert(projectId, job).execute();
+ return; // SUCCEEDED
+ } catch (GoogleJsonResponseException e) {
+ if (errorExtractor.itemAlreadyExists(e)) {
+ return; // SUCCEEDED
+ }
+ // ignore and retry
+ LOG.warn("Ignore the error and retry inserting the job.", e);
+ lastException = e;
+ } catch (IOException e) {
+ // ignore and retry
+ LOG.warn("Ignore the error and retry inserting the job.", e);
+ lastException = e;
+ }
+ } while (nextBackOff(sleeper, backoff));
+ throw new IOException(
+ String.format(
+ "Unable to insert job: %s, aborting after %d retries.",
+ jobId, MAX_LOAD_JOB_RPC_ATTEMPTS),
+ lastException);
+ }
+
+ @VisibleForTesting
+ Status pollJobStatus(
+ String projectId,
+ String jobId,
+ Sleeper sleeper,
+ BackOff backoff) throws InterruptedException {
+ do {
+ try {
+ JobStatus status = client.jobs().get(projectId, jobId).execute().getStatus();
+ if (status != null && status.getState() != null && status.getState().equals("DONE")) {
+ if (status.getErrorResult() != null) {
+ return Status.FAILED;
+ } else if (status.getErrors() != null && !status.getErrors().isEmpty()) {
+ return Status.FAILED;
+ } else {
+ return Status.SUCCEEDED;
+ }
+ }
+ // The job is not DONE, wait longer and retry.
+ } catch (IOException e) {
+ // ignore and retry
+ LOG.warn("Ignore the error and retry polling job status.", e);
+ }
+ } while (nextBackOff(sleeper, backoff));
+ LOG.warn("Unable to poll job status: {}, aborting after {} retries.",
+ jobId, MAX_LOAD_JOB_POLL_RETRIES);
+ return Status.UNKNOWN;
+ }
+
+ /**
+ * Identical to {@link BackOffUtils#next} but without checked IOException.
+ * @throws InterruptedException
+ */
+ private boolean nextBackOff(Sleeper sleeper, BackOff backoff) throws InterruptedException {
+ try {
+ return BackOffUtils.next(sleeper, backoff);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a97d79e5/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/io/BigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/io/BigQueryIOTest.java b/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/io/BigQueryIOTest.java
index 51c6556..0d084af 100644
--- a/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/io/BigQueryIOTest.java
+++ b/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/io/BigQueryIOTest.java
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import com.google.api.client.util.Data;
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
+import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableReference;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
@@ -31,10 +33,14 @@ import com.google.cloud.dataflow.sdk.io.BigQueryIO.Write.CreateDisposition;
import com.google.cloud.dataflow.sdk.io.BigQueryIO.Write.WriteDisposition;
import com.google.cloud.dataflow.sdk.options.BigQueryOptions;
import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory;
+import com.google.cloud.dataflow.sdk.testing.ExpectedLogs;
import com.google.cloud.dataflow.sdk.testing.RunnableOnService;
import com.google.cloud.dataflow.sdk.testing.TestPipeline;
import com.google.cloud.dataflow.sdk.transforms.Create;
+import com.google.cloud.dataflow.sdk.util.BigQueryServices;
+import com.google.cloud.dataflow.sdk.util.BigQueryServices.Status;
import com.google.cloud.dataflow.sdk.util.CoderUtils;
+import com.google.common.collect.ImmutableList;
import org.hamcrest.Matchers;
import org.junit.Assert;
@@ -43,8 +49,15 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
+import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
/**
* Tests for BigQueryIO.
@@ -52,8 +65,96 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class BigQueryIOTest {
+ private static class FakeBigQueryServices implements BigQueryServices {
+
+ private Object[] startLoadJobReturns;
+ private Object[] pollJobStatusReturns;
+
+ /**
+ * Sets the return values for the mock {@link LoadService#startLoadJob}.
+ *
+ * <p>Throws if the {@link Object} is a {@link Exception}, returns otherwise.
+ */
+ private FakeBigQueryServices startLoadJobReturns(Object... startLoadJobReturns) {
+ this.startLoadJobReturns = startLoadJobReturns;
+ return this;
+ }
+
+ /**
+ * Sets the return values for the mock {@link LoadService#pollJobStatus}.
+ *
+ * <p>Throws if the {@link Object} is a {@link Exception}, returns otherwise.
+ */
+ private FakeBigQueryServices pollJobStatusReturns(Object... pollJobStatusReturns) {
+ this.pollJobStatusReturns = pollJobStatusReturns;
+ return this;
+ }
+
+ @Override
+ public LoadService getLoadService(BigQueryOptions bqOptions) {
+ return new FakeLoadService(startLoadJobReturns, pollJobStatusReturns);
+ }
+
+ private static class FakeLoadService implements BigQueryServices.LoadService {
+
+ private Object[] startLoadJobReturns;
+ private Object[] pollJobStatusReturns;
+ private int startLoadJobCallsCount;
+ private int pollJobStatusCallsCount;
+
+ public FakeLoadService(Object[] startLoadJobReturns, Object[] pollJobStatusReturns) {
+ this.startLoadJobReturns = startLoadJobReturns;
+ this.pollJobStatusReturns = pollJobStatusReturns;
+ this.startLoadJobCallsCount = 0;
+ this.pollJobStatusCallsCount = 0;
+ }
+
+ @Override
+ public void startLoadJob(String jobId, JobConfigurationLoad loadConfig)
+ throws InterruptedException, IOException {
+ if (startLoadJobCallsCount < startLoadJobReturns.length) {
+ Object ret = startLoadJobReturns[startLoadJobCallsCount++];
+ if (ret instanceof IOException) {
+ throw (IOException) ret;
+ } else if (ret instanceof InterruptedException) {
+ throw (InterruptedException) ret;
+ } else {
+ return;
+ }
+ } else {
+ throw new RuntimeException(
+ "Exceeded expected number of calls: " + startLoadJobReturns.length);
+ }
+ }
+
+ @Override
+ public Status pollJobStatus(String projectId, String jobId) throws InterruptedException {
+ if (pollJobStatusCallsCount < pollJobStatusReturns.length) {
+ Object ret = pollJobStatusReturns[pollJobStatusCallsCount++];
+ if (ret instanceof Status) {
+ return (Status) ret;
+ } else if (ret instanceof InterruptedException) {
+ throw (InterruptedException) ret;
+ } else {
+ throw new RuntimeException("Unexpected return type: " + ret.getClass());
+ }
+ } else {
+ throw new RuntimeException(
+ "Exceeded expected number of calls: " + pollJobStatusReturns.length);
+ }
+ }
+ }
+ }
+
@Rule
public ExpectedException thrown = ExpectedException.none();
+ @Rule public ExpectedLogs logged = ExpectedLogs.none(BigQueryIO.class);
+ @Rule
+ public TemporaryFolder testFolder = new TemporaryFolder();
+ @Mock
+ public BigQueryServices.LoadService mockBqLoadService;
+
+ private BigQueryOptions bqOptions;
private void checkReadTableObject(
BigQueryIO.Read.Bound bound, String project, String dataset, String table) {
@@ -93,10 +194,10 @@ public class BigQueryIOTest {
BigQueryIO.Write.Bound bound, String project, String dataset, String table,
TableSchema schema, CreateDisposition createDisposition,
WriteDisposition writeDisposition, boolean validate) {
- assertEquals(project, bound.table.getProjectId());
- assertEquals(dataset, bound.table.getDatasetId());
- assertEquals(table, bound.table.getTableId());
- assertEquals(schema, bound.schema);
+ assertEquals(project, bound.getTable().getProjectId());
+ assertEquals(dataset, bound.getTable().getDatasetId());
+ assertEquals(table, bound.getTable().getTableId());
+ assertEquals(schema, bound.getSchema());
assertEquals(createDisposition, bound.createDisposition);
assertEquals(writeDisposition, bound.writeDisposition);
assertEquals(validate, bound.validate);
@@ -104,8 +205,11 @@ public class BigQueryIOTest {
@Before
public void setUp() {
- BigQueryOptions options = PipelineOptionsFactory.as(BigQueryOptions.class);
- options.setProject("defaultProject");
+ bqOptions = PipelineOptionsFactory.as(BigQueryOptions.class);
+ bqOptions.setProject("defaultProject");
+ bqOptions.setTempLocation(testFolder.getRoot().getAbsolutePath() + "/BigQueryIOTest/");
+
+ MockitoAnnotations.initMocks(this);
}
@Test
@@ -223,6 +327,67 @@ public class BigQueryIOTest {
}
@Test
+ public void testCustomSink() throws Exception {
+ FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
+ .startLoadJobReturns("done", "done", "done")
+ .pollJobStatusReturns(Status.FAILED, Status.FAILED, Status.SUCCEEDED);
+
+ Pipeline p = TestPipeline.create(bqOptions);
+ p.apply(Create.of(
+ new TableRow().set("name", "a").set("number", 1),
+ new TableRow().set("name", "b").set("number", 2),
+ new TableRow().set("name", "c").set("number", 3)))
+ .setCoder(TableRowJsonCoder.of())
+ .apply(BigQueryIO.Write.to("project-id:dataset-id.table-id")
+ .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
+ .withSchema(new TableSchema().setFields(
+ ImmutableList.of(
+ new TableFieldSchema().setName("name").setType("STRING"),
+ new TableFieldSchema().setName("number").setType("INTEGER"))))
+ .withTestServices(fakeBqServices)
+ .withoutValidation());
+ p.run();
+
+ logged.verifyInfo("Starting BigQuery load job");
+ logged.verifyInfo("Previous load jobs failed, retrying.");
+ File tempDir = new File(bqOptions.getTempLocation());
+ assertEquals(0, tempDir.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.isFile();
+ }}).length);
+ }
+
+ @Test
+ public void testCustomSinkUnknown() throws Exception {
+ FakeBigQueryServices fakeBqServices = new FakeBigQueryServices()
+ .startLoadJobReturns("done", "done")
+ .pollJobStatusReturns(Status.FAILED, Status.UNKNOWN);
+
+ Pipeline p = TestPipeline.create(bqOptions);
+ p.apply(Create.of(
+ new TableRow().set("name", "a").set("number", 1),
+ new TableRow().set("name", "b").set("number", 2),
+ new TableRow().set("name", "c").set("number", 3)))
+ .setCoder(TableRowJsonCoder.of())
+ .apply(BigQueryIO.Write.to("project-id:dataset-id.table-id")
+ .withCreateDisposition(CreateDisposition.CREATE_NEVER)
+ .withTestServices(fakeBqServices)
+ .withoutValidation());
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("Failed to poll the load job status.");
+ p.run();
+
+ File tempDir = new File(bqOptions.getTempLocation());
+ assertEquals(0, tempDir.listFiles(new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.isFile();
+ }}).length);
+ }
+
+ @Test
public void testBuildSink() {
BigQueryIO.Write.Bound bound = BigQueryIO.Write.named("WriteMyTable")
.to("foo.com:project:somedataset.sometable");
@@ -334,7 +499,6 @@ public class BigQueryIOTest {
null, CreateDisposition.CREATE_IF_NEEDED, WriteDisposition.WRITE_EMPTY);
}
-
private void testWriteValidatesDataset(boolean streaming) {
BigQueryOptions options = PipelineOptionsFactory.as(BigQueryOptions.class);
options.setProject("someproject");
@@ -351,15 +515,11 @@ public class BigQueryIOTest {
thrown.expectMessage(
Matchers.either(Matchers.containsString("Unable to confirm BigQuery dataset presence"))
.or(Matchers.containsString("BigQuery dataset not found for table")));
- try {
- p.apply(Create.<TableRow>of().withCoder(TableRowJsonCoder.of()))
- .apply(BigQueryIO.Write.named("WriteMyTable")
- .to(tableRef)
- .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
- .withSchema(new TableSchema()));
- } finally {
- Assert.assertEquals("someproject", tableRef.getProjectId());
- }
+ p.apply(Create.<TableRow>of().withCoder(TableRowJsonCoder.of()))
+ .apply(BigQueryIO.Write.named("WriteMyTable")
+ .to(tableRef)
+ .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
+ .withSchema(new TableSchema()));
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/a97d79e5/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImplTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImplTest.java b/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImplTest.java
new file mode 100644
index 0000000..200e8a7
--- /dev/null
+++ b/sdks/java/core/src/test/java/com/google/cloud/dataflow/sdk/util/BigQueryServicesImplTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.dataflow.sdk.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.api.client.googleapis.json.GoogleJsonError;
+import com.google.api.client.googleapis.json.GoogleJsonError.ErrorInfo;
+import com.google.api.client.googleapis.json.GoogleJsonErrorContainer;
+import com.google.api.client.http.LowLevelHttpResponse;
+import com.google.api.client.json.GenericJson;
+import com.google.api.client.json.Json;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.testing.http.MockHttpTransport;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.util.BackOff;
+import com.google.api.client.util.Sleeper;
+import com.google.api.services.bigquery.Bigquery;
+import com.google.api.services.bigquery.model.ErrorProto;
+import com.google.api.services.bigquery.model.Job;
+import com.google.api.services.bigquery.model.JobConfigurationLoad;
+import com.google.api.services.bigquery.model.JobStatus;
+import com.google.api.services.bigquery.model.TableReference;
+import com.google.cloud.dataflow.sdk.testing.ExpectedLogs;
+import com.google.cloud.dataflow.sdk.testing.FastNanoClockAndSleeper;
+import com.google.common.collect.ImmutableList;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Tests for {@link BigQueryServicesImpl}.
+ */
+@RunWith(JUnit4.class)
+public class BigQueryServicesImplTest {
+ @Rule public ExpectedException thrown = ExpectedException.none();
+ @Rule public ExpectedLogs expectedLogs = ExpectedLogs.none(BigQueryServicesImpl.class);
+ @Mock private LowLevelHttpResponse response;
+ private Bigquery bigquery;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.initMocks(this);
+
+ // A mock transport that lets us mock the API responses.
+ MockHttpTransport transport =
+ new MockHttpTransport.Builder()
+ .setLowLevelHttpRequest(
+ new MockLowLevelHttpRequest() {
+ @Override
+ public LowLevelHttpResponse execute() throws IOException {
+ return response;
+ }
+ })
+ .build();
+
+ // A sample BigQuery API client that uses default JsonFactory and RetryHttpInitializer.
+ bigquery =
+ new Bigquery.Builder(
+ transport, Transport.getJsonFactory(), new RetryHttpRequestInitializer())
+ .build();
+ }
+
+ /**
+ * Tests that {@link BigQueryServicesImpl.LoadServiceImpl#startLoadJob} succeeds.
+ */
+ @Test
+ public void testStartLoadJobSucceeds() throws IOException, InterruptedException {
+ Job testJob = new Job();
+ when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
+ when(response.getStatusCode()).thenReturn(200);
+ when(response.getContent()).thenReturn(toStream(testJob));
+
+ TableReference ref = new TableReference();
+ ref.setProjectId("projectId");
+ JobConfigurationLoad loadConfig = new JobConfigurationLoad();
+ loadConfig.setDestinationTable(ref);
+
+ Sleeper sleeper = new FastNanoClockAndSleeper();
+ BackOff backoff = new AttemptBoundedExponentialBackOff(
+ 5 /* attempts */, 1000 /* initialIntervalMillis */);
+ BigQueryServicesImpl.LoadServiceImpl loadService =
+ new BigQueryServicesImpl.LoadServiceImpl(bigquery);
+ loadService.startLoadJob("jobId", loadConfig, sleeper, backoff);
+ verify(response, times(1)).getStatusCode();
+ verify(response, times(1)).getContent();
+ verify(response, times(1)).getContentType();
+ }
+
+ /**
+ * Tests that {@link BigQueryServicesImpl.LoadServiceImpl#startLoadJob} succeeds
+ * with an already exist job.
+ */
+ @Test
+ public void testStartLoadJobSucceedsAlreadyExists() throws IOException, InterruptedException {
+ when(response.getStatusCode()).thenReturn(409); // 409 means already exists
+
+ TableReference ref = new TableReference();
+ ref.setProjectId("projectId");
+ JobConfigurationLoad loadConfig = new JobConfigurationLoad();
+ loadConfig.setDestinationTable(ref);
+
+ Sleeper sleeper = new FastNanoClockAndSleeper();
+ BackOff backoff = new AttemptBoundedExponentialBackOff(
+ 5 /* attempts */, 1000 /* initialIntervalMillis */);
+ BigQueryServicesImpl.LoadServiceImpl loadService =
+ new BigQueryServicesImpl.LoadServiceImpl(bigquery);
+ loadService.startLoadJob("jobId", loadConfig, sleeper, backoff);
+
+ verify(response, times(1)).getStatusCode();
+ verify(response, times(1)).getContent();
+ verify(response, times(1)).getContentType();
+ }
+
+ /**
+ * Tests that {@link BigQueryServicesImpl.LoadServiceImpl#startLoadJob} succeeds with a retry.
+ */
+ @Test
+ public void testStartLoadJobRetry() throws IOException, InterruptedException {
+ Job testJob = new Job();
+
+ // First response is 403 rate limited, second response has valid payload.
+ when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
+ when(response.getStatusCode()).thenReturn(403).thenReturn(200);
+ when(response.getContent())
+ .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)))
+ .thenReturn(toStream(testJob));
+
+ TableReference ref = new TableReference();
+ ref.setProjectId("projectId");
+ JobConfigurationLoad loadConfig = new JobConfigurationLoad();
+ loadConfig.setDestinationTable(ref);
+
+ Sleeper sleeper = new FastNanoClockAndSleeper();
+ BackOff backoff = new AttemptBoundedExponentialBackOff(
+ 5 /* attempts */, 1000 /* initialIntervalMillis */);
+ BigQueryServicesImpl.LoadServiceImpl loadService =
+ new BigQueryServicesImpl.LoadServiceImpl(bigquery);
+ loadService.startLoadJob("jobId", loadConfig, sleeper, backoff);
+ verify(response, times(2)).getStatusCode();
+ verify(response, times(2)).getContent();
+ verify(response, times(2)).getContentType();
+ }
+
+ /**
+ * Tests that {@link BigQueryServicesImpl.LoadServiceImpl#pollJobStatus} succeeds.
+ */
+ @Test
+ public void testPollJobStatusSucceeds() throws IOException, InterruptedException {
+ Job testJob = new Job();
+ testJob.setStatus(new JobStatus().setState("DONE"));
+
+ when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
+ when(response.getStatusCode()).thenReturn(200);
+ when(response.getContent()).thenReturn(toStream(testJob));
+
+ BigQueryServicesImpl.LoadServiceImpl loadService =
+ new BigQueryServicesImpl.LoadServiceImpl(bigquery);
+ BigQueryServices.Status status =
+ loadService.pollJobStatus("projectId", "jobId", Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);
+
+ assertEquals(BigQueryServices.Status.SUCCEEDED, status);
+ verify(response, times(1)).getStatusCode();
+ verify(response, times(1)).getContent();
+ verify(response, times(1)).getContentType();
+ }
+
+ /**
+ * Tests that {@link BigQueryServicesImpl.LoadServiceImpl#pollJobStatus} fails.
+ */
+ @Test
+ public void testPollJobStatusFailed() throws IOException, InterruptedException {
+ Job testJob = new Job();
+ testJob.setStatus(new JobStatus().setState("DONE").setErrorResult(new ErrorProto()));
+
+ when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
+ when(response.getStatusCode()).thenReturn(200);
+ when(response.getContent()).thenReturn(toStream(testJob));
+
+ BigQueryServicesImpl.LoadServiceImpl loadService =
+ new BigQueryServicesImpl.LoadServiceImpl(bigquery);
+ BigQueryServices.Status status =
+ loadService.pollJobStatus("projectId", "jobId", Sleeper.DEFAULT, BackOff.ZERO_BACKOFF);
+
+ assertEquals(BigQueryServices.Status.FAILED, status);
+ verify(response, times(1)).getStatusCode();
+ verify(response, times(1)).getContent();
+ verify(response, times(1)).getContentType();
+ }
+
+ /**
+ * Tests that {@link BigQueryServicesImpl.LoadServiceImpl#pollJobStatus} returns UNKNOWN.
+ */
+ @Test
+ public void testPollJobStatusUnknown() throws IOException, InterruptedException {
+ Job testJob = new Job();
+ testJob.setStatus(new JobStatus());
+
+ when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
+ when(response.getStatusCode()).thenReturn(200);
+ when(response.getContent()).thenReturn(toStream(testJob));
+
+ BigQueryServicesImpl.LoadServiceImpl loadService =
+ new BigQueryServicesImpl.LoadServiceImpl(bigquery);
+ BigQueryServices.Status status =
+ loadService.pollJobStatus("projectId", "jobId", Sleeper.DEFAULT, BackOff.STOP_BACKOFF);
+
+ assertEquals(BigQueryServices.Status.UNKNOWN, status);
+ verify(response, times(1)).getStatusCode();
+ verify(response, times(1)).getContent();
+ verify(response, times(1)).getContentType();
+ }
+
+ /** A helper to wrap a {@link GenericJson} object in a content stream. */
+ private static InputStream toStream(GenericJson content) throws IOException {
+ return new ByteArrayInputStream(JacksonFactory.getDefaultInstance().toByteArray(content));
+ }
+
+ /** A helper that generates the error JSON payload that Google APIs produce. */
+ private static GoogleJsonErrorContainer errorWithReasonAndStatus(String reason, int status) {
+ ErrorInfo info = new ErrorInfo();
+ info.setReason(reason);
+ info.setDomain("global");
+ // GoogleJsonError contains one or more ErrorInfo objects; our utiities read the first one.
+ GoogleJsonError error = new GoogleJsonError();
+ error.setErrors(ImmutableList.of(info));
+ error.setCode(status);
+ // The actual JSON response is an error container.
+ GoogleJsonErrorContainer container = new GoogleJsonErrorContainer();
+ container.setError(error);
+ return container;
+ }
+}
+
[2/2] incubator-beam git commit: Closes #48
Posted by dh...@apache.org.
Closes #48
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/936d028f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/936d028f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/936d028f
Branch: refs/heads/master
Commit: 936d028f05a3835fdef6b15e7e00424780eda213
Parents: d2b659b a97d79e
Author: Dan Halperin <dh...@google.com>
Authored: Mon Apr 11 15:15:59 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Apr 11 15:15:59 2016 -0700
----------------------------------------------------------------------
.../cloud/dataflow/sdk/io/BigQueryIO.java | 378 ++++++++++++++-----
.../dataflow/sdk/util/BigQueryServices.java | 61 +++
.../dataflow/sdk/util/BigQueryServicesImpl.java | 196 ++++++++++
.../cloud/dataflow/sdk/io/BigQueryIOTest.java | 192 +++++++++-
.../sdk/util/BigQueryServicesImplTest.java | 262 +++++++++++++
5 files changed, 975 insertions(+), 114 deletions(-)
----------------------------------------------------------------------