You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/07/11 19:55:23 UTC
[1/2] incubator-beam git commit: Migrated IO display data tests must
be runner-filesystem agnostic
Repository: incubator-beam
Updated Branches:
refs/heads/master 1c5858b86 -> c5744ccf2
Migrated IO display data tests must be runner-filesystem agnostic
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1117a033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1117a033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1117a033
Branch: refs/heads/master
Commit: 1117a0331485d654ceb0d6ce424f681d48fab5ea
Parents: 1c5858b
Author: Scott Wegner <sw...@google.com>
Authored: Fri Jul 8 15:53:43 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Jul 11 12:55:18 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 20 +++++++++++++++++---
.../org/apache/beam/sdk/io/BigQueryIOTest.java | 14 ++++++++++----
.../java/org/apache/beam/sdk/io/TextIOTest.java | 20 +++++++++++++++++---
3 files changed, 44 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1117a033/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
index 047e7d0..026724a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/AvroIOTest.java
@@ -29,10 +29,12 @@ import static org.junit.Assert.assertTrue;
import org.apache.beam.sdk.coders.AvroCoder;
import org.apache.beam.sdk.coders.DefaultCoder;
import org.apache.beam.sdk.io.AvroIO.Write.Bound;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.RunnableOnService;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestPipelineOptions;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
@@ -47,6 +49,8 @@ import org.apache.avro.Schema;
import org.apache.avro.file.DataFileReader;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.reflect.Nullable;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -69,6 +73,11 @@ public class AvroIOTest {
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
+ @BeforeClass
+ public static void setupClass() {
+ IOChannelUtils.registerStandardIOFactories(TestPipeline.testingPipelineOptions());
+ }
+
@Test
public void testReadWithoutValidationFlag() throws Exception {
AvroIO.Read.Bound<GenericRecord> read = AvroIO.Read.from("gs://bucket/foo*/baz");
@@ -313,11 +322,16 @@ public class AvroIOTest {
@Test
@Category(RunnableOnService.class)
- public void testPrimitiveWriteDisplayData() {
- DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
+ @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient")
+ public void testPrimitiveWriteDisplayData() throws IOException {
+ PipelineOptions options = DisplayDataEvaluator.getDefaultOptions();
+ String tempRoot = options.as(TestPipelineOptions.class).getTempRoot();
+ String outputPath = IOChannelUtils.getFactory(tempRoot).resolve(tempRoot, "foo");
+
+ DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(options);
AvroIO.Write.Bound<?> write = AvroIO.Write
- .to("foo")
+ .to(outputPath)
.withSchema(Schema.create(Schema.Type.STRING))
.withoutValidation();
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1117a033/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java
index 78d950e..0d1a9f8 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/BigQueryIOTest.java
@@ -94,6 +94,7 @@ import org.hamcrest.CoreMatchers;
import org.hamcrest.Matchers;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -643,8 +644,9 @@ public class BigQueryIOTest implements Serializable {
@Test
@Category(RunnableOnService.class)
+ @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient")
public void testTableSourcePrimitiveDisplayData() throws IOException, InterruptedException {
- DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(bqOptions);
+ DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
BigQueryIO.Read.Bound read = BigQueryIO.Read
.from("project:dataset.tableId")
.withTestServices(new FakeBigQueryServices()
@@ -659,8 +661,9 @@ public class BigQueryIOTest implements Serializable {
@Test
@Category(RunnableOnService.class)
+ @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient")
public void testQuerySourcePrimitiveDisplayData() throws IOException, InterruptedException {
- DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(bqOptions);
+ DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
BigQueryIO.Read.Bound read = BigQueryIO.Read
.fromQuery("foobar")
.withTestServices(new FakeBigQueryServices()
@@ -684,20 +687,23 @@ public class BigQueryIOTest implements Serializable {
@Test
@Category(RunnableOnService.class)
+ @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient")
public void testBatchSinkPrimitiveDisplayData() throws IOException, InterruptedException {
testSinkPrimitiveDisplayData(/* streaming: */ false);
}
@Test
@Category(RunnableOnService.class)
+ @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient")
public void testStreamingSinkPrimitiveDisplayData() throws IOException, InterruptedException {
testSinkPrimitiveDisplayData(/* streaming: */ true);
}
private void testSinkPrimitiveDisplayData(boolean streaming) throws IOException,
InterruptedException {
- bqOptions.as(StreamingOptions.class).setStreaming(streaming);
- DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(bqOptions);
+ PipelineOptions options = TestPipeline.testingPipelineOptions();
+ options.as(StreamingOptions.class).setStreaming(streaming);
+ DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(options);
BigQueryIO.Write.Bound write = BigQueryIO.Write
.to("project:dataset.table")
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/1117a033/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
index 28e9ea4..babb50a 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/TextIOTest.java
@@ -42,12 +42,14 @@ import org.apache.beam.sdk.io.BoundedSource.BoundedReader;
import org.apache.beam.sdk.io.TextIO.CompressionType;
import org.apache.beam.sdk.io.TextIO.TextSource;
import org.apache.beam.sdk.options.GcsOptions;
+import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.testing.NeedsRunner;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.RunnableOnService;
import org.apache.beam.sdk.testing.SourceTestUtils;
import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.testing.TestPipelineOptions;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
@@ -59,6 +61,8 @@ import org.apache.beam.sdk.values.PCollection;
import com.google.common.collect.ImmutableList;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -101,6 +105,11 @@ public class TextIOTest {
@Rule public TemporaryFolder tmpFolder = new TemporaryFolder();
@Rule public ExpectedException expectedException = ExpectedException.none();
+ @BeforeClass
+ public static void setupClass() {
+ IOChannelUtils.registerStandardIOFactories(TestPipeline.testingPipelineOptions());
+ }
+
<T> void runTestRead(T[] expected, Coder<T> coder) throws Exception {
File tmpFile = tmpFolder.newFile("file.txt");
String filename = tmpFile.getPath();
@@ -336,14 +345,19 @@ public class TextIOTest {
@Test
@Category(RunnableOnService.class)
- public void testPrimitiveWriteDisplayData() {
+ @Ignore("[BEAM-436] DirectRunner RunnableOnService tempLocation configuration insufficient")
+ public void testPrimitiveWriteDisplayData() throws IOException {
+ PipelineOptions options = DisplayDataEvaluator.getDefaultOptions();
+ String tempRoot = options.as(TestPipelineOptions.class).getTempRoot();
+ String outputPath = IOChannelUtils.getFactory(tempRoot).resolve(tempRoot, "foobar");
+
DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
- TextIO.Write.Bound<?> write = TextIO.Write.to("foobar");
+ TextIO.Write.Bound<?> write = TextIO.Write.to(outputPath);
Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(write);
assertThat("TextIO.Write should include the file prefix in its primitive display data",
- displayData, hasItem(hasDisplayItem(hasValue(startsWith("foobar")))));
+ displayData, hasItem(hasDisplayItem(hasValue(startsWith(outputPath)))));
}
@Test
[2/2] incubator-beam git commit: Closes #616
Posted by dh...@apache.org.
Closes #616
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c5744ccf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c5744ccf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c5744ccf
Branch: refs/heads/master
Commit: c5744ccf2d287db7edcee4c6aa4add2ac84621e3
Parents: 1c5858b 1117a03
Author: Dan Halperin <dh...@google.com>
Authored: Mon Jul 11 12:55:19 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Jul 11 12:55:19 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 20 +++++++++++++++++---
.../org/apache/beam/sdk/io/BigQueryIOTest.java | 14 ++++++++++----
.../java/org/apache/beam/sdk/io/TextIOTest.java | 20 +++++++++++++++++---
3 files changed, 44 insertions(+), 10 deletions(-)
----------------------------------------------------------------------