You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/08/02 18:07:36 UTC
[4/5] incubator-beam git commit: Some fixups to tests
Some fixups to tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ab20823c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ab20823c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ab20823c
Branch: refs/heads/master
Commit: ab20823cda024ed3061034774c2a896d2ddc0671
Parents: 70f394b
Author: Dan Halperin <dh...@google.com>
Authored: Fri Jul 29 16:51:09 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Aug 2 11:07:29 2016 -0700
----------------------------------------------------------------------
.../dataflow/options/DataflowPipelineOptions.java | 1 -
.../beam/runners/dataflow/DataflowRunnerTest.java | 3 ++-
.../options/DataflowPipelineOptionsTest.java | 3 +++
.../runners/dataflow/util/GcsPathValidatorTest.java | 15 +++++++++++++--
.../org/apache/beam/sdk/options/GcpOptionsTest.java | 2 ++
5 files changed, 20 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ab20823c/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
index 0c5bf47..841741f 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java
@@ -33,7 +33,6 @@ import org.apache.beam.sdk.options.PubsubOptions;
import org.apache.beam.sdk.options.StreamingOptions;
import org.apache.beam.sdk.options.Validation;
import org.apache.beam.sdk.util.IOChannelUtils;
-import org.apache.beam.sdk.util.gcsfs.GcsPath;
import com.google.common.base.MoreObjects;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ab20823c/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index a29cdc9..704410d 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -229,7 +229,7 @@ public class DataflowRunnerTest {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
options.setRunner(DataflowRunner.class);
options.setProject(PROJECT_ID);
- options.setGcpTempLocation("gs://somebucket/some/path");
+ options.setTempLocation("gs://somebucket/some/path");
// Set FILES_PROPERTY to empty to prevent a default value calculated from classpath.
options.setFilesToStage(new LinkedList<String>());
options.setDataflowClient(buildMockDataflow(jobCaptor));
@@ -617,6 +617,7 @@ public class DataflowRunnerTest {
GcsUtil mockGcsUtil =
buildMockGcsUtil(true /* temp bucket exists */, false /* staging bucket exists */);
DataflowPipelineOptions options = buildPipelineOptions(jobCaptor);
+ options.setGcpTempLocation(options.getTempLocation()); // bypass validation for GcpTempLocation
options.setGcsUtil(mockGcsUtil);
options.setStagingLocation("gs://non-existent-bucket/location");
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ab20823c/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
----------------------------------------------------------------------
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
index b5ee5e9..c0422ee 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptionsTest.java
@@ -25,6 +25,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.ResetDateTimeProvider;
import org.apache.beam.sdk.testing.RestoreSystemProperties;
import org.apache.beam.sdk.util.IOChannelUtils;
+import org.apache.beam.sdk.util.NoopPathValidator;
import org.junit.Rule;
import org.junit.Test;
@@ -109,6 +110,7 @@ public class DataflowPipelineOptionsTest {
public void testDefaultToTempLocation() {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
IOChannelUtils.registerStandardIOFactories(options);
+ options.setPathValidatorClass(NoopPathValidator.class);
options.setTempLocation("gs://temp_location");
assertEquals("gs://temp_location", options.getGcpTempLocation());
assertEquals("gs://temp_location/staging", options.getStagingLocation());
@@ -118,6 +120,7 @@ public class DataflowPipelineOptionsTest {
public void testDefaultToGcpTempLocation() {
DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
IOChannelUtils.registerStandardIOFactories(options);
+ options.setPathValidatorClass(NoopPathValidator.class);
options.setTempLocation("gs://temp_location");
options.setGcpTempLocation("gs://gcp_temp_location");
assertEquals("gs://gcp_temp_location/staging", options.getStagingLocation());
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ab20823c/sdks/java/core/src/test/java/org/apache/beam/runners/dataflow/util/GcsPathValidatorTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/runners/dataflow/util/GcsPathValidatorTest.java b/sdks/java/core/src/test/java/org/apache/beam/runners/dataflow/util/GcsPathValidatorTest.java
index d101627..8913916 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/runners/dataflow/util/GcsPathValidatorTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/runners/dataflow/util/GcsPathValidatorTest.java
@@ -21,8 +21,11 @@ import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.when;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.options.GcsOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.runners.PipelineRunner;
import org.apache.beam.sdk.util.GcsPathValidator;
import org.apache.beam.sdk.util.GcsUtil;
import org.apache.beam.sdk.util.TestCredential;
@@ -45,12 +48,20 @@ public class GcsPathValidatorTest {
@Mock private GcsUtil mockGcsUtil;
private GcsPathValidator validator;
+ private class FakeRunner extends PipelineRunner<PipelineResult> {
+ @Override
+ public PipelineResult run(Pipeline pipeline) {
+ throw new UnsupportedOperationException();
+ }
+ }
+
@Before
public void setUp() throws Exception {
MockitoAnnotations.initMocks(this);
when(mockGcsUtil.bucketExists(any(GcsPath.class))).thenReturn(true);
when(mockGcsUtil.isGcsPatternSupported(anyString())).thenCallRealMethod();
GcsOptions options = PipelineOptionsFactory.as(GcsOptions.class);
+ options.setRunner(FakeRunner.class);
options.setGcpCredential(new TestCredential());
options.setGcsUtil(mockGcsUtil);
validator = GcsPathValidator.fromOptions(options);
@@ -65,7 +76,7 @@ public class GcsPathValidatorTest {
public void testInvalidFilePattern() {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage(
- "DirectRunner expected a valid 'gs://' path but was given '/local/path'");
+ "FakeRunner expected a valid 'gs://' path but was given '/local/path'");
validator.validateInputFilePatternSupported("/local/path");
}
@@ -87,7 +98,7 @@ public class GcsPathValidatorTest {
public void testInvalidOutputPrefix() {
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage(
- "DirectRunner expected a valid 'gs://' path but was given '/local/path'");
+ "FakeRunner expected a valid 'gs://' path but was given '/local/path'");
validator.validateOutputFilePrefixSupported("/local/path");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ab20823c/sdks/java/core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java
index c0f65d8..22359dc 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/options/GcpOptionsTest.java
@@ -26,6 +26,7 @@ import static org.mockito.Mockito.when;
import org.apache.beam.sdk.options.GcpOptions.DefaultProjectFactory;
import org.apache.beam.sdk.testing.RestoreSystemProperties;
+import org.apache.beam.sdk.util.NoopPathValidator;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;
@@ -117,6 +118,7 @@ public class GcpOptionsTest {
GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class);
String tempLocation = "gs://bucket";
options.setTempLocation(tempLocation);
+ options.as(GcsOptions.class).setPathValidatorClass(NoopPathValidator.class);
assertEquals(tempLocation, options.getGcpTempLocation());
}