You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/08/02 18:07:35 UTC
[3/5] incubator-beam git commit: DataflowRunner: test path validation
DataflowRunner: test path validation
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1f87b848
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1f87b848
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1f87b848
Branch: refs/heads/master
Commit: 1f87b84838f53c49151cfadcd20dcf6890f8491f
Parents: d73e614
Author: Dan Halperin <dh...@google.com>
Authored: Fri Jul 29 15:30:16 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Aug 2 11:07:28 2016 -0700
----------------------------------------------------------------------
.../runners/dataflow/DataflowRunnerTest.java | 36 ++++++++++++++++++++
1 file changed, 36 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1f87b848/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index fe288ad..d3f2f75 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -18,6 +18,7 @@
package org.apache.beam.runners.dataflow;
import static org.apache.beam.sdk.util.WindowedValue.valueInGlobalWindow;
+
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@@ -62,6 +63,7 @@ import org.apache.beam.sdk.options.PipelineOptions.CheckEnabled;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.runners.TransformTreeNode;
import org.apache.beam.sdk.runners.dataflow.TestCountingSource;
+import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFnTester;
import org.apache.beam.sdk.transforms.PTransform;
@@ -70,6 +72,7 @@ import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.GcsUtil;
+import org.apache.beam.sdk.util.NoopCredentialFactory;
import org.apache.beam.sdk.util.NoopPathValidator;
import org.apache.beam.sdk.util.ReleaseInfo;
import org.apache.beam.sdk.util.TestCredential;
@@ -88,6 +91,7 @@ import com.google.api.services.dataflow.Dataflow;
import com.google.api.services.dataflow.model.DataflowPackage;
import com.google.api.services.dataflow.model.Job;
import com.google.api.services.dataflow.model.ListJobsResponse;
+import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -235,6 +239,38 @@ public class DataflowRunnerTest {
}
@Test
+ public void testPathValidation() {
+ String[] args = new String[] {
+ "--runner=DataflowRunner",
+ "--tempLocation=/tmp/not/a/gs/path",
+ "--project=test-project",
+ "--credentialFactoryClass=" + NoopCredentialFactory.class.getCanonicalName(),
+ };
+
+ try {
+ TestPipeline.fromOptions(PipelineOptionsFactory.fromArgs(args).create());
+ fail();
+ } catch (RuntimeException e) {
+ assertThat(
+ Throwables.getStackTraceAsString(e),
+ containsString("DataflowRunner requires gcpTempLocation"));
+ }
+ }
+
+ @Test
+ public void testPathValidatorOverride() {
+ String[] args = new String[] {
+ "--runner=DataflowRunner",
+ "--tempLocation=/tmp/testing",
+ "--project=test-project",
+ "--credentialFactoryClass=" + NoopCredentialFactory.class.getCanonicalName(),
+ "--pathValidatorClass=" + NoopPathValidator.class.getCanonicalName(),
+ };
+ // Should not crash, because gcpTempLocation should get set from tempLocation
+ TestPipeline.fromOptions(PipelineOptionsFactory.fromArgs(args).create());
+ }
+
+ @Test
public void testFromOptionsWithUppercaseConvertsToLowercase() throws Exception {
String mixedCase = "ThisJobNameHasMixedCase";
ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);