You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ec...@apache.org on 2019/01/15 12:40:32 UTC
[beam] 02/03: Comment schema choices
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 1fe87911591940f49d2db95d3cda5a906a04fd96
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Tue Jan 15 13:35:52 2019 +0100
Comment schema choices
---
.../structuredstreaming/translation/batch/DatasetSourceBatch.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
index c4cfeaf..2a13d98 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
@@ -96,6 +96,9 @@ public class DatasetSourceBatch implements DataSourceV2, ReadSupport {
@Override
public StructType readSchema() {
// TODO: find a way to extend schema with a WindowedValue schema
+ // we use a binary schema for now because:
+ // using a empty schema raises a indexOutOfBoundsException
+ // using a NullType schema stores null in the elements
StructField[] array = new StructField[1];
StructField binaryStructField = StructField
.apply("binaryStructField", DataTypes.BinaryType, true, Metadata.empty());