You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ju...@apache.org on 2016/10/12 00:29:29 UTC
arrow git commit: ARROW-333: Make writers update their internal
schema even when no data is written
Repository: arrow
Updated Branches:
refs/heads/master 994aa5a90 -> caa843bda
ARROW-333: Make writers update their internal schema even when no data is written
Make PromotableWriter predefine writers when asked
Author: Julien Le Dem <ju...@dremio.com>
Closes #170 from julienledem/promotable_writer_preset and squashes the following commits:
972eb9c [Julien Le Dem] ARROW-333: Make writers update their internal schema even when no data is written Make PromotableWriter predefine writers when asked
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/caa843bd
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/caa843bd
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/caa843bd
Branch: refs/heads/master
Commit: caa843bdaf395b915a739bf5e1d6c5eabe1f4693
Parents: 994aa5a
Author: Julien Le Dem <ju...@dremio.com>
Authored: Tue Oct 11 17:29:25 2016 -0700
Committer: Julien Le Dem <ju...@dremio.com>
Committed: Tue Oct 11 17:29:25 2016 -0700
----------------------------------------------------------------------
.../src/main/codegen/templates/MapWriters.java | 15 +++++++++
.../src/main/codegen/templates/UnionWriter.java | 24 +++++++++++++++
.../vector/complex/impl/PromotableWriter.java | 14 ++++-----
.../complex/writer/TestComplexWriter.java | 32 ++++++++++++++++++--
4 files changed, 76 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/caa843bd/java/vector/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 696bbf6..51327b4 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -112,6 +112,11 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
}
writer.setPosition(idx());
fields.put(name.toLowerCase(), writer);
+ } else {
+ if (writer instanceof PromotableWriter) {
+ // ensure writers are initialized
+ ((PromotableWriter)writer).getWriter(MinorType.MAP);
+ }
}
return writer;
}
@@ -149,6 +154,11 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
}
writer.setPosition(idx());
fields.put(name.toLowerCase(), writer);
+ } else {
+ if (writer instanceof PromotableWriter) {
+ // ensure writers are initialized
+ ((PromotableWriter)writer).getWriter(MinorType.LIST);
+ }
}
return writer;
}
@@ -210,6 +220,11 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
}
writer.setPosition(idx());
fields.put(name.toLowerCase(), writer);
+ } else {
+ if (writer instanceof PromotableWriter) {
+ // ensure writers are initialized
+ ((PromotableWriter)writer).getWriter(MinorType.${upperName});
+ }
}
return writer;
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/caa843bd/java/vector/src/main/codegen/templates/UnionWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 460ec1c..efb66f1 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -25,6 +25,8 @@
package org.apache.arrow.vector.complex.impl;
<#include "/@includes/vv_imports.ftl" />
+import org.apache.arrow.vector.complex.writer.BaseWriter;
+import org.apache.arrow.vector.types.Types.MinorType;
/*
* This class is generated using freemarker and the ${.template_name} template.
@@ -100,6 +102,28 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
return getListWriter();
}
+ BaseWriter getWriter(MinorType minorType) {
+ switch (minorType) {
+ case MAP:
+ return getMapWriter();
+ case LIST:
+ return getListWriter();
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.class?starts_with("Decimal")>
+ case ${name?upper_case}:
+ return get${name}Writer();
+ </#if>
+ </#list>
+ </#list>
+ default:
+ throw new UnsupportedOperationException("Unknown type: " + minorType);
+ }
+ }
+
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
http://git-wip-us.apache.org/repos/asf/arrow/blob/caa843bd/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index c282688..94ff82c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -94,19 +94,19 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
protected FieldWriter getWriter(MinorType type) {
if (state == State.UNION) {
- return writer;
- }
- if (state == State.UNTYPED) {
+ ((UnionWriter)writer).getWriter(type);
+ } else if (state == State.UNTYPED) {
if (type == null) {
+ // ???
return null;
}
ValueVector v = listVector.addOrGetVector(type).getVector();
v.allocateNew();
setWriter(v);
writer.setPosition(position);
- }
- if (type != this.type) {
- return promoteToUnion();
+ } else if (type != this.type) {
+ promoteToUnion();
+ ((UnionWriter)writer).getWriter(type);
}
return writer;
}
@@ -133,7 +133,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
unionVector.addVector((FieldVector)tp.getTo());
writer = new UnionWriter(unionVector);
writer.setPosition(idx());
- for (int i = 0; i < idx(); i++) {
+ for (int i = 0; i <= idx(); i++) {
unionVector.getMutator().setType(i, vector.getMinorType());
}
vector = null;
http://git-wip-us.apache.org/repos/asf/arrow/blob/caa843bd/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 398aea9..9419f88 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -45,6 +45,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Int;
import org.apache.arrow.vector.types.pojo.ArrowType.Union;
import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.Text;
import org.junit.Assert;
import org.junit.Test;
@@ -362,11 +363,38 @@ public class TestComplexWriter {
MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
for (int i = 0; i < 100; i++) {
rootReader.setPosition(i);
- Assert.assertEquals(i, rootReader.reader("a").readLong().intValue());
+ FieldReader reader = rootReader.reader("a");
+ Long value = reader.readLong();
+ Assert.assertNotNull("index: " + i, value);
+ Assert.assertEquals(i, value.intValue());
}
for (int i = 100; i < 200; i++) {
rootReader.setPosition(i);
- Assert.assertEquals(Integer.toString(i), rootReader.reader("a").readText().toString());
+ FieldReader reader = rootReader.reader("a");
+ Text value = reader.readText();
+ Assert.assertEquals(Integer.toString(i), value.toString());
}
}
+
+ /**
+ * Even without writing to the writer, the union schema is created correctly
+ */
+ @Test
+ public void promotableWriterSchema() {
+ MapVector parent = new MapVector("parent", allocator, null);
+ ComplexWriter writer = new ComplexWriterImpl("root", parent);
+ MapWriter rootWriter = writer.rootAsMap();
+ BigIntWriter bigIntWriter = rootWriter.bigInt("a");
+ VarCharWriter varCharWriter = rootWriter.varChar("a");
+
+ Field field = parent.getField().getChildren().get(0).getChildren().get(0);
+ Assert.assertEquals("a", field.getName());
+ Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeType());
+
+ Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeType());
+ Int intType = (Int) field.getChildren().get(0).getType();
+ Assert.assertEquals(64, intType.getBitWidth());
+ Assert.assertTrue(intType.getIsSigned());
+ Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeType());
+ }
}
\ No newline at end of file