You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ec...@apache.org on 2019/01/08 10:58:28 UTC
[beam] 01/03: Add a dummy schema for reader
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 57edb20619dc223a2b528d2fe7ac94f3df2d957d
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Tue Jan 8 10:27:28 2019 +0100
Add a dummy schema for reader
---
.../translation/batch/DatasetSourceBatch.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
index 1c8a888..c1a93e3 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
@@ -38,6 +38,9 @@ import org.apache.spark.sql.sources.v2.ReadSupport;
import org.apache.spark.sql.sources.v2.reader.DataSourceReader;
import org.apache.spark.sql.sources.v2.reader.InputPartition;
import org.apache.spark.sql.sources.v2.reader.InputPartitionReader;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;
/**
@@ -94,7 +97,11 @@ public class DatasetSourceBatch<T> implements DataSourceV2, ReadSupport {
@Override
public StructType readSchema() {
- return new StructType();
+ StructField[] array = new StructField[1];
+ StructField dummyStructField = StructField
+ .apply("dummyStructField", DataTypes.NullType, true, Metadata.empty());
+ array[0] = dummyStructField;
+ return new StructType(array);
}
@Override