You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vo...@apache.org on 2020/05/25 22:34:50 UTC
[drill] 01/04: DRILL-7741: Columns are missing when using
convert_from function
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 7bac742aa0ba8289325fd7c71bae3236087a6e48
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Fri May 15 14:30:47 2020 +0300
DRILL-7741: Columns are missing when using convert_from function
closes #2081
---
.../physical/impl/project/ProjectRecordBatch.java | 10 ++++++++++
.../exec/vector/complex/writer/TestJsonReader.java | 22 ++++++++++++++++++++++
2 files changed, 32 insertions(+)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 62bd0c6..948f283 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -332,6 +332,16 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
return IterOutcome.OK_NEW_SCHEMA;
}
+ @Override
+ protected IterOutcome getFinalOutcome(boolean hasMoreRecordInBoundary) {
+ // In a case of complex writers vectors are added at runtime, so the schema
+ // may change (e.g. when a batch contains new column(s) not present in previous batches)
+ if (complexWriters != null) {
+ return IterOutcome.OK_NEW_SCHEMA;
+ }
+ return super.getFinalOutcome(hasMoreRecordInBoundary);
+ }
+
private void setupNewSchema(RecordBatch incomingBatch, int configuredBatchSize) {
memoryManager = new ProjectMemoryManager(configuredBatchSize);
memoryManager.init(incomingBatch, ProjectRecordBatch.this);
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 04bc67d..3fae12b 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -782,4 +782,26 @@ public class TestJsonReader extends BaseTestQuery {
resetSessionOption(ExecConstants.ENABLE_UNION_TYPE_KEY);
}
}
+
+ @Test
+ public void testConvertFromJson() throws Exception {
+ String fileName = "table.tsv";
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), fileName)))) {
+ for (int i = 0; i < JSONRecordReader.DEFAULT_ROWS_PER_BATCH; i++) {
+ writer.write("{\"id\":\"1\"}\n");
+ }
+ writer.write("{\"id\":\"2\",\"v\":[\"abc\"]}");
+ }
+
+ String sql = "SELECT t.m.id AS id, t.m.v[0] v FROM \n" +
+ "(SELECT convert_from(columns[0], 'json') AS m FROM dfs.`%s`) t\n" +
+ "where t.m.id='2'";
+
+ testBuilder()
+ .sqlQuery(sql, fileName)
+ .unOrdered()
+ .baselineColumns("id", "v")
+ .baselineValues("2", "abc")
+ .go();
+ }
}