You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sm...@apache.org on 2016/08/18 23:30:38 UTC
[5/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead
of MaterializedField
ARROW-259: Use Flatbuffer Field type instead of MaterializedField
Remove MaterializedField, MajorType, RepeatedTypes
Add code to convert from FlatBuf representation to Pojo
also adds tests to test the conversion
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e7e399db
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e7e399db
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e7e399db
Branch: refs/heads/master
Commit: e7e399db5fc6913e67426514279f81766a0778d2
Parents: 246a126
Author: Steven Phillips <st...@dremio.com>
Authored: Tue May 24 13:38:09 2016 -0700
Committer: Steven Phillips <st...@dremio.com>
Committed: Thu Aug 18 16:29:27 2016 -0700
----------------------------------------------------------------------
format/Message.fbs | 22 +-
header | 16 +
java/format/pom.xml | 163 ++++
java/memory/pom.xml | 2 +-
java/pom.xml | 3 +-
java/vector/pom.xml | 7 +-
java/vector/src/main/codegen/config.fmpp | 1 +
.../vector/src/main/codegen/data/ArrowTypes.tdd | 80 ++
.../src/main/codegen/data/ValueVectorTypes.tdd | 59 +-
.../src/main/codegen/includes/vv_imports.ftl | 4 +
.../codegen/templates/AbstractFieldReader.java | 8 +-
.../codegen/templates/AbstractFieldWriter.java | 10 +-
.../AbstractPromotableFieldWriter.java | 4 -
.../src/main/codegen/templates/ArrowType.java | 129 +++
.../src/main/codegen/templates/BaseReader.java | 5 +-
.../src/main/codegen/templates/BaseWriter.java | 3 +-
.../main/codegen/templates/BasicTypeHelper.java | 539 -------------
.../main/codegen/templates/ComplexCopier.java | 18 +-
.../main/codegen/templates/ComplexReaders.java | 72 +-
.../main/codegen/templates/ComplexWriters.java | 30 +-
.../codegen/templates/FixedValueVectors.java | 94 ++-
.../codegen/templates/HolderReaderImpl.java | 98 +--
.../src/main/codegen/templates/ListWriters.java | 234 ------
.../src/main/codegen/templates/MapWriters.java | 42 +-
.../src/main/codegen/templates/NullReader.java | 23 +-
.../codegen/templates/NullableValueVectors.java | 104 ++-
.../codegen/templates/RepeatedValueVectors.java | 421 ----------
.../main/codegen/templates/UnionListWriter.java | 23 +-
.../src/main/codegen/templates/UnionReader.java | 28 +-
.../src/main/codegen/templates/UnionVector.java | 105 ++-
.../src/main/codegen/templates/UnionWriter.java | 16 +-
.../main/codegen/templates/ValueHolders.java | 43 +-
.../templates/VariableLengthVectors.java | 73 +-
.../arrow/vector/BaseDataValueVector.java | 5 +-
.../apache/arrow/vector/BaseValueVector.java | 31 +-
.../java/org/apache/arrow/vector/BitVector.java | 43 +-
.../org/apache/arrow/vector/ObjectVector.java | 220 -----
.../apache/arrow/vector/ValueHolderHelper.java | 203 -----
.../org/apache/arrow/vector/ValueVector.java | 10 +-
.../apache/arrow/vector/VectorDescriptor.java | 83 --
.../org/apache/arrow/vector/ZeroVector.java | 30 +-
.../vector/complex/AbstractContainerVector.java | 49 +-
.../arrow/vector/complex/AbstractMapVector.java | 47 +-
.../vector/complex/BaseRepeatedValueVector.java | 63 +-
.../vector/complex/ContainerVectorLike.java | 43 -
.../apache/arrow/vector/complex/ListVector.java | 89 +-
.../apache/arrow/vector/complex/MapVector.java | 97 +--
.../vector/complex/RepeatedListVector.java | 427 ----------
.../arrow/vector/complex/RepeatedMapVector.java | 584 --------------
.../vector/complex/RepeatedValueVector.java | 2 +-
.../vector/complex/impl/AbstractBaseReader.java | 19 +-
.../vector/complex/impl/AbstractBaseWriter.java | 16 +-
.../vector/complex/impl/ComplexWriterImpl.java | 22 +-
.../vector/complex/impl/PromotableWriter.java | 48 +-
.../complex/impl/RepeatedListReaderImpl.java | 145 ----
.../complex/impl/RepeatedMapReaderImpl.java | 192 -----
.../impl/SingleLikeRepeatedMapReaderImpl.java | 89 --
.../complex/impl/SingleListReaderImpl.java | 14 +-
.../complex/impl/SingleMapReaderImpl.java | 10 +-
.../vector/complex/impl/UnionListReader.java | 19 +-
.../arrow/vector/holders/ObjectHolder.java | 38 -
.../arrow/vector/holders/UnionHolder.java | 7 +-
.../arrow/vector/types/MaterializedField.java | 217 -----
.../org/apache/arrow/vector/types/Types.java | 596 +++++++++++---
.../apache/arrow/vector/types/pojo/Field.java | 105 +++
.../apache/arrow/vector/types/pojo/Schema.java | 74 ++
.../arrow/vector/util/ByteFunctionHelpers.java | 50 --
.../arrow/vector/util/CoreDecimalUtility.java | 91 ---
.../arrow/vector/util/DecimalUtility.java | 802 +++++++++----------
.../arrow/vector/util/MapWithOrdinal.java | 12 +
.../apache/arrow/vector/TestDecimalVector.java | 63 ++
.../TestOversizedAllocationForValueVector.java | 11 +-
.../apache/arrow/vector/TestUnionVector.java | 5 +-
.../apache/arrow/vector/TestValueVector.java | 137 +---
.../complex/impl/TestPromotableWriter.java | 7 +-
.../complex/writer/TestComplexWriter.java | 270 +++++++
.../apache/arrow/vector/pojo/TestConvert.java | 80 ++
77 files changed, 2464 insertions(+), 5180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/format/Message.fbs
----------------------------------------------------------------------
diff --git a/format/Message.fbs b/format/Message.fbs
index 3f688c1..2928207 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -1,10 +1,13 @@
-namespace apache.arrow.flatbuf;
+namespace org.apache.arrow.flatbuf;
/// ----------------------------------------------------------------------
/// Logical types and their metadata (if any)
///
/// These are stored in the flatbuffer in the Type union below
+table Null {
+}
+
/// A Tuple in the flatbuffer metadata is the same as an Arrow Struct
/// (according to the physical memory layout). We used Tuple here as Struct is
/// a reserved word in Flatbuffers
@@ -45,10 +48,22 @@ table Decimal {
scale: int;
}
+table Date {
+}
+
+table Time {
+}
+
table Timestamp {
timezone: string;
}
+table IntervalDay {
+}
+
+table IntervalYear {
+}
+
table JSONScalar {
dense:bool=true;
}
@@ -58,13 +73,18 @@ table JSONScalar {
/// add new logical types to Type without breaking backwards compatibility
union Type {
+ Null,
Int,
FloatingPoint,
Binary,
Utf8,
Bool,
Decimal,
+ Date,
+ Time,
Timestamp,
+ IntervalDay,
+ IntervalYear,
List,
Tuple,
Union,
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/header
----------------------------------------------------------------------
diff --git a/header b/header
new file mode 100644
index 0000000..70665d1
--- /dev/null
+++ b/header
@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
new file mode 100644
index 0000000..ea27a30
--- /dev/null
+++ b/java/format/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<modelVersion>4.0.0</modelVersion>
+
+<parent>
+ <artifactId>arrow-java-root</artifactId>
+ <groupId>org.apache.arrow</groupId>
+ <version>0.1-decimal</version>
+</parent>
+
+<artifactId>arrow-format</artifactId>
+<packaging>jar</packaging>
+<name>Arrow Format</name>
+
+ <properties>
+ <fbs.version>1.2.0-3f79e055</fbs.version>
+ <maven-compiler-plugin.version>3.3</maven-compiler-plugin.version>
+ <maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
+ <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.vlkan</groupId>
+ <artifactId>flatbuffers</artifactId>
+ <version>${fbs.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <extensions>
+ <!-- provides os.detected.classifier (i.e. linux-x86_64, osx-x86_64) property -->
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+ <version>${os-maven-plugin.version}</version>
+ </extension>
+ </extensions>
+
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>${maven-dependency-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy-flatc</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>com.vlkan</groupId>
+ <artifactId>flatc-${os.detected.classifier}</artifactId>
+ <version>${fbs.version}</version>
+ <type>exe</type>
+ <overWrite>true</overWrite>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.4.0</version>
+ <executions>
+ <execution>
+ <id>script-chmod</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <executable>chmod</executable>
+ <arguments>
+ <argument>+x</argument>
+ <argument>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>generate-sources</phase>
+ <configuration>
+ <executable>${project.build.directory}/flatc-${os.detected.classifier}-${fbs.version}.exe</executable>
+ <arguments>
+ <argument>-j</argument>
+ <argument>-o</argument>
+ <argument>target/generated-sources/</argument>
+ <argument>../../format/Message.fbs</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>com.mycila</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <version>2.3</version>
+ <configuration>
+ <header>${basedir}/../../header</header>
+ <includes>
+ <include>**/*.java</include>
+ </includes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>format</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.9.1</version>
+ <executions>
+ <execution>
+ <id>add-sources-as-resources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+
+</build>
+</project>
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 44332f5..12ff4c8 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -15,7 +15,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.1-decimal</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>arrow-memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 71f59ca..92ab109 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.1-decimal</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -465,6 +465,7 @@
</dependencies>
<modules>
+ <module>format</module>
<module>memory</module>
<module>vector</module>
</modules>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index df53892..fac788c 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -15,7 +15,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>0.1-decimal</version>
</parent>
<artifactId>vector</artifactId>
<name>vectors</name>
@@ -24,6 +24,11 @@
<dependency>
<groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-format</artifactId>
+ <version>0.1-decimal</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/config.fmpp b/java/vector/src/main/codegen/config.fmpp
index 663677c..6d92ba8 100644
--- a/java/vector/src/main/codegen/config.fmpp
+++ b/java/vector/src/main/codegen/config.fmpp
@@ -17,6 +17,7 @@
data: {
# TODO: Rename to ~valueVectorModesAndTypes for clarity.
vv: tdd(../data/ValueVectorTypes.tdd),
+ arrowTypes: tdd(../data/ArrowTypes.tdd)
}
freemarkerLinks: {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/data/ArrowTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
new file mode 100644
index 0000000..4ab7f85
--- /dev/null
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ types: [
+ {
+ name: "Null",
+ fields: []
+ },
+ {
+ name: "Tuple",
+ fields: []
+ },
+ {
+ name: "List",
+ fields: []
+ },
+ {
+ name: "Union",
+ fields: []
+ },
+ {
+ name: "Int",
+ fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
+ },
+ {
+ name: "FloatingPoint",
+ fields: [{name: precision, type: int}]
+ },
+ {
+ name: "Utf8",
+ fields: []
+ },
+ {
+ name: "Binary",
+ fields: []
+ },
+ {
+ name: "Bool",
+ fields: []
+ },
+ {
+ name: "Decimal",
+ fields: [{name: "precision", type: int}, {name: "scale", type: int}]
+ },
+ {
+ name: "Date",
+ fields: []
+ },
+ {
+ name: "Time",
+ fields: []
+ },
+ {
+ name: "Timestamp",
+ fields: [{name: "timezone", type: "String"}]
+ },
+ {
+ name: "IntervalDay",
+ fields: []
+ },
+ {
+ name: "IntervalYear",
+ fields: []
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index e747c30..421dd7e 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -17,8 +17,7 @@
{
modes: [
{name: "Optional", prefix: "Nullable"},
- {name: "Required", prefix: ""},
- {name: "Repeated", prefix: "Repeated"}
+ {name: "Required", prefix: ""}
],
types: [
{
@@ -61,9 +60,8 @@
{ class: "Int", valueHolder: "IntHolder"},
{ class: "UInt4", valueHolder: "UInt4Holder" },
{ class: "Float4", javaType: "float" , boxedType: "Float", fields: [{name: "value", type: "float"}]},
- { class: "Time", javaType: "int", friendlyType: "DateTime" },
{ class: "IntervalYear", javaType: "int", friendlyType: "Period" }
- { class: "Decimal9", maxPrecisionDigits: 9, friendlyType: "BigDecimal", fields: [{name:"value", type:"int"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
+ { class: "Time", javaType: "int", friendlyType: "DateTime" }
]
},
{
@@ -78,15 +76,11 @@
{ class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
{ class: "Date", javaType: "long", friendlyType: "DateTime" },
{ class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
- { class: "Decimal18", maxPrecisionDigits: 18, friendlyType: "BigDecimal", fields: [{name:"value", type:"long"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] },
- <#--
- { class: "Money", maxPrecisionDigits: 2, scale: 1, },
- -->
]
},
{
major: "Fixed",
- width: 12,
+ width: 8,
javaType: "ArrowBuf",
boxedType: "ArrowBuf",
minor: [
@@ -96,51 +90,11 @@
{
major: "Fixed",
width: 16,
- javaType: "ArrowBuf"
- boxedType: "ArrowBuf",
- minor: [
- { class: "Interval", daysOffset: 4, millisecondsOffset: 8, friendlyType: "Period", fields: [ {name: "months", type: "int"}, {name: "days", type:"int"}, {name: "milliseconds", type:"int"}] }
- ]
- },
- {
- major: "Fixed",
- width: 12,
- javaType: "ArrowBuf",
- boxedType: "ArrowBuf",
- minor: [
- <#--
- { class: "TimeTZ" },
- { class: "Interval" }
- -->
- { class: "Decimal28Dense", maxPrecisionDigits: 28, nDecimalDigits: 3, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 16,
- javaType: "ArrowBuf",
- boxedType: "ArrowBuf",
-
- minor: [
- { class: "Decimal38Dense", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 24,
- javaType: "ArrowBuf",
- boxedType: "ArrowBuf",
- minor: [
- { class: "Decimal38Sparse", maxPrecisionDigits: 38, nDecimalDigits: 6, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
- ]
- },
- {
- major: "Fixed",
- width: 20,
javaType: "ArrowBuf",
boxedType: "ArrowBuf",
+
minor: [
- { class: "Decimal28Sparse", maxPrecisionDigits: 28, nDecimalDigits: 5, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
+ { class: "Decimal", maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal", fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}] }
]
},
{
@@ -151,8 +105,7 @@
fields: [{name: "start", type: "int"}, {name: "end", type: "int"}, {name: "buffer", type: "ArrowBuf"}],
minor: [
{ class: "VarBinary" , friendlyType: "byte[]" },
- { class: "VarChar" , friendlyType: "Text" },
- { class: "Var16Char" , friendlyType: "String" }
+ { class: "VarChar" , friendlyType: "Text" }
]
},
{
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index 2d808b1..9b4b79b 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -17,6 +17,8 @@ import com.google.common.collect.ObjectArrays;
import com.google.common.base.Charsets;
import com.google.common.collect.ObjectArrays;
+import com.google.flatbuffers.FlatBufferBuilder;
+
import com.google.common.base.Preconditions;
import io.netty.buffer.*;
@@ -25,6 +27,8 @@ import org.apache.commons.lang3.ArrayUtils;
import org.apache.arrow.memory.*;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.*;
+import org.apache.arrow.vector.types.pojo.*;
+import org.apache.arrow.vector.types.pojo.ArrowType.*;
import org.apache.arrow.vector.types.*;
import org.apache.arrow.vector.*;
import org.apache.arrow.vector.holders.*;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/AbstractFieldReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldReader.java b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
index b83dba2..e0d0fc9 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldReader.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldReader.java
@@ -41,7 +41,13 @@ abstract class AbstractFieldReader extends AbstractBaseReader implements FieldRe
return true;
}
- <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
+ @Override
+ public Field getField() {
+ fail("getField");
+ return null;
+ }
+
+ <#list ["Object", "BigDecimal", "Integer", "Long", "Boolean",
"Character", "DateTime", "Period", "Double", "Float",
"Text", "String", "Byte", "Short", "byte[]"] as friendlyType>
<#assign safeType=friendlyType />
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 6ee9dad..de076fc 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -31,10 +31,6 @@ package org.apache.arrow.vector.complex.impl;
*/
@SuppressWarnings("unused")
abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWriter {
- AbstractFieldWriter(FieldWriter parent) {
- super(parent);
- }
-
@Override
public void start() {
throw new IllegalStateException(String.format("You tried to start when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
@@ -62,9 +58,15 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr
fail("${name}");
}
+ <#if minor.class == "Decimal">
+ public void writeDecimal(int start, ArrowBuf buffer) {
+ fail("${name}");
+ }
+ <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
fail("${name}");
}
+ </#if>
</#list></#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 549dbf1..7e60320 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -37,10 +37,6 @@ package org.apache.arrow.vector.complex.impl;
*/
@SuppressWarnings("unused")
abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
- AbstractPromotableFieldWriter(FieldWriter parent) {
- super(parent);
- }
-
/**
* Retrieve the FieldWriter, promoting if it is not a FieldWriter of the specified type
* @param type
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ArrowType.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ArrowType.java b/java/vector/src/main/codegen/templates/ArrowType.java
new file mode 100644
index 0000000..6dfaf21
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ArrowType.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.arrow.flatbuf.Field;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+
+import java.util.Objects;
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/types/pojo/ArrowType.java" />
+
+
+<#include "/@includes/license.ftl" />
+package org.apache.arrow.vector.types.pojo;
+
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Type;
+
+import java.util.Objects;
+
+public abstract class ArrowType {
+
+ public abstract byte getTypeType();
+ public abstract int getType(FlatBufferBuilder builder);
+
+
+ <#list arrowTypes.types as type>
+ <#assign name = type.name>
+ <#assign fields = type.fields>
+ public static class ${name} extends ArrowType {
+ public static final byte TYPE_TYPE = Type.${name};
+ <#if type.fields?size == 0>
+ public static final ${name} INSTANCE = new ${name}();
+ </#if>
+
+ <#list fields as field>
+ <#assign fieldName = field.name>
+ <#assign fieldType = field.type>
+ ${fieldType} ${fieldName};
+ </#list>
+
+ <#if type.fields?size != 0>
+ public ${type.name}(<#list type.fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+ <#list type.fields as field>
+ this.${field.name} = ${field.name};
+ </#list>
+ }
+ </#if>
+
+ @Override
+ public byte getTypeType() {
+ return TYPE_TYPE;
+ }
+
+ @Override
+ public int getType(FlatBufferBuilder builder) {
+ org.apache.arrow.flatbuf.${type.name}.start${type.name}(builder);
+ <#list type.fields as field>
+ org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, <#if field.type == "String">builder.createString(${field.name})<#else>${field.name}</#if>);
+ </#list>
+ return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
+ }
+
+ <#list fields as field>
+ public ${field.type} get${field.name?cap_first}() {
+ return ${field.name};
+ }
+ </#list>
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ${type.name})) {
+ return false;
+ }
+ <#if type.fields?size == 0>
+ return true;
+ <#else>
+ ${type.name} that = (${type.name}) obj;
+ return
+ <#list type.fields as field>Objects.equals(this.${field.name}, that.${field.name}) <#if field_has_next>&&<#else>;</#if>
+ </#list>
+ </#if>
+ }
+ }
+ </#list>
+
+ public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.apache.arrow.flatbuf.Field field) {
+ switch(field.typeType()) {
+ <#list arrowTypes.types as type>
+ <#assign name = type.name>
+ <#assign nameLower = type.name?lower_case>
+ <#assign fields = type.fields>
+ case Type.${type.name}:
+ org.apache.arrow.flatbuf.${type.name} ${nameLower}Type = (org.apache.arrow.flatbuf.${type.name}) field.type(new org.apache.arrow.flatbuf.${type.name}());
+ return new ${type.name}(<#list type.fields as field>${nameLower}Type.${field.name}()<#if field_has_next>, </#if></#list>);
+ </#list>
+ default:
+ throw new UnsupportedOperationException("Unsupported type: " + field.typeType());
+ }
+ }
+
+ public static Int getInt(org.apache.arrow.flatbuf.Field field) {
+ org.apache.arrow.flatbuf.Int intType = (org.apache.arrow.flatbuf.Int) field.type(new org.apache.arrow.flatbuf.Int());
+ return new Int(intType.bitWidth(), intType.isSigned());
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/BaseReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/BaseReader.java b/java/vector/src/main/codegen/templates/BaseReader.java
index 8f12b1d..72fea58 100644
--- a/java/vector/src/main/codegen/templates/BaseReader.java
+++ b/java/vector/src/main/codegen/templates/BaseReader.java
@@ -30,8 +30,8 @@ package org.apache.arrow.vector.complex.reader;
@SuppressWarnings("unused")
public interface BaseReader extends Positionable{
- MajorType getType();
- MaterializedField getField();
+ Field getField();
+ MinorType getMinorType();
void reset();
void read(UnionHolder holder);
void read(int index, UnionHolder holder);
@@ -60,7 +60,6 @@ public interface BaseReader extends Positionable{
public interface ScalarReader extends
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> ${name}Reader, </#list></#list>
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first /> Repeated${name}Reader, </#list></#list>
BaseReader {}
interface ComplexReader{
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/BaseWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/BaseWriter.java b/java/vector/src/main/codegen/templates/BaseWriter.java
index 299b238..08bd39e 100644
--- a/java/vector/src/main/codegen/templates/BaseWriter.java
+++ b/java/vector/src/main/codegen/templates/BaseWriter.java
@@ -31,12 +31,11 @@ package org.apache.arrow.vector.complex.writer;
*/
@SuppressWarnings("unused")
public interface BaseWriter extends AutoCloseable, Positionable {
- FieldWriter getParent();
int getValueCapacity();
public interface MapWriter extends BaseWriter {
- MaterializedField getField();
+ Field getField();
/**
* Whether this writer is a map writer and is empty (has no children).
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/BasicTypeHelper.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/BasicTypeHelper.java b/java/vector/src/main/codegen/templates/BasicTypeHelper.java
deleted file mode 100644
index 0bae715..0000000
--- a/java/vector/src/main/codegen/templates/BasicTypeHelper.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-<@pp.changeOutputFile name="/org/apache/arrow/vector/util/BasicTypeHelper.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector.util;
-
-<#include "/@includes/vv_imports.ftl" />
-import org.apache.arrow.vector.complex.UnionVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.util.CallBack;
-
-public class BasicTypeHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BasicTypeHelper.class);
-
- private static final int WIDTH_ESTIMATE = 50;
-
- // Default length when casting to varchar : 65536 = 2^16
- // This only defines an absolute maximum for values, setting
- // a high value like this will not inflate the size for small values
- public static final int VARCHAR_DEFAULT_CAST_LEN = 65536;
-
- protected static String buildErrorMessage(final String operation, final MinorType type, final DataMode mode) {
- return String.format("Unable to %s for minor type [%s] and mode [%s]", operation, type, mode);
- }
-
- protected static String buildErrorMessage(final String operation, final MajorType type) {
- return buildErrorMessage(operation, type.getMinorType(), type.getMode());
- }
-
- public static int getSize(MajorType major) {
- switch (major.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
- minor.class?substring(0, 3) == "PRO" ||
- minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
- </#list>
-</#list>
-// case FIXEDCHAR: return major.getWidth();
-// case FIXED16CHAR: return major.getWidth();
-// case FIXEDBINARY: return major.getWidth();
- }
- throw new UnsupportedOperationException(buildErrorMessage("get size", major));
- }
-
- public static ValueVector getNewVector(String name, BufferAllocator allocator, MajorType type, CallBack callback){
- MaterializedField field = MaterializedField.create(name, type);
- return getNewVector(field, allocator, callback);
- }
-
-
- public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionVector.class;
- case MAP:
- switch (mode) {
- case OPTIONAL:
- case REQUIRED:
- return MapVector.class;
- case REPEATED:
- return RepeatedMapVector.class;
- }
-
- case LIST:
- switch (mode) {
- case REPEATED:
- return RepeatedListVector.class;
- case REQUIRED:
- case OPTIONAL:
- return ListVector.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}Vector.class;
- case OPTIONAL:
- return Nullable${minor.class}Vector.class;
- case REPEATED:
- return Repeated${minor.class}Vector.class;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT :
- return ObjectVector.class ;
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type, mode));
- }
- public static Class<?> getReaderClassName( MinorType type, DataMode mode, boolean isSingularRepeated){
- switch (type) {
- case MAP:
- switch (mode) {
- case REQUIRED:
- if (!isSingularRepeated)
- return SingleMapReaderImpl.class;
- else
- return SingleLikeRepeatedMapReaderImpl.class;
- case REPEATED:
- return RepeatedMapReaderImpl.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- return SingleListReaderImpl.class;
- case REPEATED:
- return RepeatedListReaderImpl.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}ReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}ReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}ReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get reader class name", type, mode));
- }
-
- public static Class<?> getWriterInterface( MinorType type, DataMode mode){
- switch (type) {
- case UNION: return UnionWriter.class;
- case MAP: return MapWriter.class;
- case LIST: return ListWriter.class;
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}: return ${minor.class}Writer.class;
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer interface", type, mode));
- }
-
- public static Class<?> getWriterImpl( MinorType type, DataMode mode){
- switch (type) {
- case UNION:
- return UnionWriter.class;
- case MAP:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return SingleMapWriter.class;
- case REPEATED:
- return RepeatedMapWriter.class;
- }
- case LIST:
- switch (mode) {
- case REQUIRED:
- case OPTIONAL:
- return UnionListWriter.class;
- case REPEATED:
- return RepeatedListWriter.class;
- }
-
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}WriterImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}WriterImpl.class;
- case REPEATED:
- return Repeated${minor.class}WriterImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get writer implementation", type, mode));
- }
-
- public static Class<?> getHolderReaderImpl( MinorType type, DataMode mode){
- switch (type) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (mode) {
- case REQUIRED:
- return ${minor.class}HolderReaderImpl.class;
- case OPTIONAL:
- return Nullable${minor.class}HolderReaderImpl.class;
- case REPEATED:
- return Repeated${minor.class}HolderReaderImpl.class;
- }
- </#list>
-</#list>
- default:
- break;
- }
- throw new UnsupportedOperationException(buildErrorMessage("get holder reader implementation", type, mode));
- }
-
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
- return getNewVector(field, allocator, null);
- }
- public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- field = field.clone();
- MajorType type = field.getType();
-
- switch (type.getMinorType()) {
-
- case UNION:
- return new UnionVector(field, allocator, callBack);
-
- case MAP:
- switch (type.getMode()) {
- case REQUIRED:
- case OPTIONAL:
- return new MapVector(field, allocator, callBack);
- case REPEATED:
- return new RepeatedMapVector(field, allocator, callBack);
- }
- case LIST:
- switch (type.getMode()) {
- case REPEATED:
- return new RepeatedListVector(field, allocator, callBack);
- case OPTIONAL:
- case REQUIRED:
- return new ListVector(field, allocator, callBack);
- }
-<#list vv. types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case}:
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Vector(field, allocator);
- case OPTIONAL:
- return new Nullable${minor.class}Vector(field, allocator);
- case REPEATED:
- return new Repeated${minor.class}Vector(field, allocator);
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- return new ObjectVector(field, allocator) ;
- default:
- break;
- }
- // All ValueVector types have been handled.
- throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
- }
-
- public static ValueHolder getValue(ValueVector vector, int index) {
- MajorType type = vector.getField().getType();
- ValueHolder holder;
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- <#if minor.class?starts_with("Var") || minor.class == "IntervalDay" || minor.class == "Interval" ||
- minor.class?starts_with("Decimal28") || minor.class?starts_with("Decimal38")>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Vector) vector).getAccessor().get(index, (${minor.class}Holder)holder);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getAccessor().get(index, (Nullable${minor.class}Holder)holder);
- }
- return holder;
- }
- <#else>
- switch (type.getMode()) {
- case REQUIRED:
- holder = new ${minor.class}Holder();
- ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
- return holder;
- case OPTIONAL:
- holder = new Nullable${minor.class}Holder();
- ((Nullable${minor.class}Holder)holder).isSet = ((Nullable${minor.class}Vector) vector).getAccessor().isSet(index);
- if (((Nullable${minor.class}Holder)holder).isSet == 1) {
- ((Nullable${minor.class}Holder)holder).value = ((Nullable${minor.class}Vector) vector).getAccessor().get(index);
- }
- return holder;
- }
- </#if>
- </#list>
-</#list>
- case GENERIC_OBJECT:
- holder = new ObjectHolder();
- ((ObjectHolder)holder).obj = ((ObjectVector) vector).getAccessor().getObject(index) ;
- break;
- }
-
- throw new UnsupportedOperationException(buildErrorMessage("get value", type));
- }
-
- public static void setValue(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- }
- return;
- }
- </#list>
-</#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- return;
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value", type));
- }
- }
-
- public static void setValueSafe(ValueVector vector, int index, ValueHolder holder) {
- MajorType type = vector.getField().getType();
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- ((${minor.class}Vector) vector).getMutator().setSafe(index, (${minor.class}Holder) holder);
- return;
- case OPTIONAL:
- if (((Nullable${minor.class}Holder) holder).isSet == 1) {
- ((Nullable${minor.class}Vector) vector).getMutator().setSafe(index, (Nullable${minor.class}Holder) holder);
- } else {
- ((Nullable${minor.class}Vector) vector).getMutator().isSafe(index);
- }
- return;
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- ((ObjectVector) vector).getMutator().setSafe(index, (ObjectHolder) holder);
- default:
- throw new UnsupportedOperationException(buildErrorMessage("set value safe", type));
- }
- }
-
- public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
- MajorType type1 = v1.getField().getType();
- MajorType type2 = v2.getField().getType();
-
- if (type1.getMinorType() != type2.getMinorType()) {
- return false;
- }
-
- switch(type1.getMinorType()) {
-<#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
- ((${minor.class}Vector) v2).getAccessor().get(v2index) )
- return true;
- break;
- </#list>
-</#list>
- default:
- break;
- }
- return false;
- }
-
- /**
- * Create a ValueHolder of MajorType.
- * @param type
- * @return
- */
- public static ValueHolder createValueHolder(MajorType type) {
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return new ${minor.class}Holder();
- case OPTIONAL:
- return new Nullable${minor.class}Holder();
- case REPEATED:
- return new Repeated${minor.class}Holder();
- }
- </#list>
- </#list>
- case GENERIC_OBJECT:
- return new ObjectHolder();
- default:
- throw new UnsupportedOperationException(buildErrorMessage("create value holder", type));
- }
- }
-
- public static boolean isNull(ValueHolder holder) {
- MajorType type = getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return true;
- case OPTIONAL:
- return ((Nullable${minor.class}Holder) holder).isSet == 0;
- case REPEATED:
- return true;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("check is null", type));
- }
- }
-
- public static ValueHolder deNullify(ValueHolder holder) {
- MajorType type = getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
-
- switch (type.getMode()) {
- case REQUIRED:
- return holder;
- case OPTIONAL:
- if( ((Nullable${minor.class}Holder) holder).isSet == 1) {
- ${minor.class}Holder newHolder = new ${minor.class}Holder();
-
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((Nullable${minor.class}Holder) holder).${field.name};
- </#list>
-
- return newHolder;
- } else {
- throw new UnsupportedOperationException("You can not convert a null value into a non-null value!");
- }
- case REPEATED:
- return holder;
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("deNullify", type));
- }
- }
-
- public static ValueHolder nullify(ValueHolder holder) {
- MajorType type = getValueHolderType(holder);
-
- switch(type.getMinorType()) {
- <#list vv.types as type>
- <#list type.minor as minor>
- case ${minor.class?upper_case} :
- switch (type.getMode()) {
- case REQUIRED:
- Nullable${minor.class}Holder newHolder = new Nullable${minor.class}Holder();
- newHolder.isSet = 1;
- <#assign fields = minor.fields!type.fields />
- <#list fields as field>
- newHolder.${field.name} = ((${minor.class}Holder) holder).${field.name};
- </#list>
- return newHolder;
- case OPTIONAL:
- return holder;
- case REPEATED:
- throw new UnsupportedOperationException("You can not convert repeated type " + type + " to nullable type!");
- }
- </#list>
- </#list>
- default:
- throw new UnsupportedOperationException(buildErrorMessage("nullify", type));
- }
- }
-
- public static MajorType getValueHolderType(ValueHolder holder) {
-
- if (0 == 1) {
- return null;
- }
- <#list vv.types as type>
- <#list type.minor as minor>
- else if (holder instanceof ${minor.class}Holder) {
- return ((${minor.class}Holder) holder).TYPE;
- } else if (holder instanceof Nullable${minor.class}Holder) {
- return ((Nullable${minor.class}Holder) holder).TYPE;
- }
- </#list>
- </#list>
-
- throw new UnsupportedOperationException("ValueHolder is not supported for 'getValueHolderType' method.");
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ComplexCopier.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexCopier.java b/java/vector/src/main/codegen/templates/ComplexCopier.java
index 3614231..a5756a4 100644
--- a/java/vector/src/main/codegen/templates/ComplexCopier.java
+++ b/java/vector/src/main/codegen/templates/ComplexCopier.java
@@ -42,13 +42,7 @@ public class ComplexCopier {
}
private static void writeValue(FieldReader reader, FieldWriter writer) {
- final DataMode m = reader.getType().getMode();
- final MinorType mt = reader.getType().getMinorType();
-
- switch(m){
- case OPTIONAL:
- case REQUIRED:
-
+ final MinorType mt = reader.getMinorType();
switch (mt) {
@@ -89,12 +83,10 @@ public class ComplexCopier {
</#if>
</#list></#list>
}
- break;
- }
}
private static FieldWriter getMapWriterForReader(FieldReader reader, MapWriter writer, String name) {
- switch (reader.getType().getMinorType()) {
+ switch (reader.getMinorType()) {
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
@@ -108,12 +100,12 @@ public class ComplexCopier {
case LIST:
return (FieldWriter) writer.list(name);
default:
- throw new UnsupportedOperationException(reader.getType().toString());
+ throw new UnsupportedOperationException(reader.getMinorType().toString());
}
}
private static FieldWriter getListWriterForReader(FieldReader reader, ListWriter writer) {
- switch (reader.getType().getMinorType()) {
+ switch (reader.getMinorType()) {
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
@@ -127,7 +119,7 @@ public class ComplexCopier {
case LIST:
return (FieldWriter) writer.list();
default:
- throw new UnsupportedOperationException(reader.getType().toString());
+ throw new UnsupportedOperationException(reader.getMinorType().toString());
}
}
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ComplexReaders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 34c6571..74a19a6 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -27,10 +27,10 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
-<#list ["", "Repeated"] as mode>
+<#list [""] as mode>
<#assign lowerName = minor.class?uncap_first />
<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-<#assign name = mode + minor.class?cap_first />
+<#assign name = minor.class?cap_first />
<#assign javaType = (minor.javaType!type.javaType) />
<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
<#assign safeType=friendlyType />
@@ -38,9 +38,9 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
<#assign hasFriendly = minor.friendlyType!"no" == "no" />
-<#list ["", "Nullable"] as nullMode>
-<#if (mode == "Repeated" && nullMode == "") || mode == "" >
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${nullMode}${name}ReaderImpl.java" />
+<#list ["Nullable"] as nullMode>
+<#if mode == "" >
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${name}ReaderImpl.java" />
<#include "/@includes/license.ftl" />
package org.apache.arrow.vector.complex.impl;
@@ -48,20 +48,20 @@ package org.apache.arrow.vector.complex.impl;
<#include "/@includes/vv_imports.ftl" />
@SuppressWarnings("unused")
-public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
+public class ${name}ReaderImpl extends AbstractFieldReader {
private final ${nullMode}${name}Vector vector;
- public ${nullMode}${name}ReaderImpl(${nullMode}${name}Vector vector){
+ public ${name}ReaderImpl(${nullMode}${name}Vector vector){
super();
this.vector = vector;
}
- public MajorType getType(){
- return vector.getField().getType();
+ public MinorType getMinorType(){
+ return vector.getMinorType();
}
- public MaterializedField getField(){
+ public Field getField(){
return vector.getField();
}
@@ -73,50 +73,13 @@ public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
</#if>
}
-
-
-
- <#if mode == "Repeated">
-
- public void copyAsValue(${minor.class?cap_first}Writer writer){
- Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer;
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public void copyAsField(String name, MapWriter writer){
- Repeated${minor.class?cap_first}WriterImpl impl = (Repeated${minor.class?cap_first}WriterImpl) writer.list(name).${lowerName}();
- impl.vector.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public int size(){
- return vector.getAccessor().getInnerValueCountAt(idx());
- }
-
- public void read(int arrayIndex, ${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), arrayIndex, h);
- }
- public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h){
- vector.getAccessor().get(idx(), arrayIndex, h);
- }
-
- public ${friendlyType} read${safeType}(int arrayIndex){
- return vector.getAccessor().getSingleObject(idx(), arrayIndex);
- }
-
-
- public List<Object> readObject(){
- return (List<Object>) (Object) vector.getAccessor().getObject(idx());
- }
-
- <#else>
-
public void copyAsValue(${minor.class?cap_first}Writer writer){
- ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer;
+ ${minor.class?cap_first}WriterImpl impl = (${minor.class?cap_first}WriterImpl) writer;
impl.vector.copyFromSafe(idx(), impl.idx(), vector);
}
public void copyAsField(String name, MapWriter writer){
- ${nullMode}${minor.class?cap_first}WriterImpl impl = (${nullMode}${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
+ ${minor.class?cap_first}WriterImpl impl = (${minor.class?cap_first}WriterImpl) writer.${lowerName}(name);
impl.vector.copyFromSafe(idx(), impl.idx(), vector);
}
@@ -141,9 +104,6 @@ public class ${nullMode}${name}ReaderImpl extends AbstractFieldReader {
public Object readObject(){
return vector.getAccessor().getObject(idx());
}
-
-
- </#if>
}
</#if>
</#list>
@@ -156,18 +116,10 @@ package org.apache.arrow.vector.complex.reader;
@SuppressWarnings("unused")
public interface ${name}Reader extends BaseReader{
- <#if mode == "Repeated">
- public int size();
- public void read(int arrayIndex, ${minor.class?cap_first}Holder h);
- public void read(int arrayIndex, Nullable${minor.class?cap_first}Holder h);
- public Object readObject(int arrayIndex);
- public ${friendlyType} read${safeType}(int arrayIndex);
- <#else>
public void read(${minor.class?cap_first}Holder h);
public void read(Nullable${minor.class?cap_first}Holder h);
public Object readObject();
public ${friendlyType} read${safeType}();
- </#if>
public boolean isSet();
public void copyAsValue(${minor.class}Writer writer);
public void copyAsField(String name, ${minor.class}Writer writer);
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ComplexWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index 8f9a6e7..3457545 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -19,8 +19,8 @@
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as mode>
-<#assign name = mode + minor.class?cap_first />
+<#list ["Nullable"] as mode>
+<#assign name = minor.class?cap_first />
<#assign eName = name />
<#assign javaType = (minor.javaType!type.javaType) />
<#assign fields = minor.fields!type.fields />
@@ -38,17 +38,16 @@ package org.apache.arrow.vector.complex.impl;
@SuppressWarnings("unused")
public class ${eName}WriterImpl extends AbstractFieldWriter {
- private final ${name}Vector.Mutator mutator;
- final ${name}Vector vector;
+ private final Nullable${name}Vector.Mutator mutator;
+ final Nullable${name}Vector vector;
- public ${eName}WriterImpl(${name}Vector vector, AbstractFieldWriter parent) {
- super(parent);
+ public ${eName}WriterImpl(Nullable${name}Vector vector) {
this.mutator = vector.getMutator();
this.vector = vector;
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return vector.getField();
}
@@ -89,12 +88,10 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
vector.getMutator().setValueCount(idx()+1);
}
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
mutator.addSafe(idx(), <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
vector.getMutator().setValueCount(idx()+1);
}
- </#if>
public void setPosition(int idx) {
super.setPosition(idx);
@@ -114,11 +111,17 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
vector.getMutator().setValueCount(idx()+1);
}
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ <#if minor.class == "Decimal">
+ public void writeDecimal(int start, ArrowBuf buffer) {
+ mutator.setSafe(idx(), 1, start, buffer);
+ vector.getMutator().setValueCount(idx()+1);
+ }
+ <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
- mutator.setSafe(idx(), <#if mode == "Nullable">1, </#if><#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+ mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
vector.getMutator().setValueCount(idx()+1);
}
+ </#if>
<#if mode == "Nullable">
@@ -128,7 +131,6 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
}
</#if>
</#if>
- </#if>
}
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/writer/${eName}Writer.java" />
@@ -141,7 +143,9 @@ package org.apache.arrow.vector.complex.writer;
public interface ${eName}Writer extends BaseWriter {
public void write(${minor.class}Holder h);
- <#if !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
+ <#if minor.class == "Decimal">
+ public void writeDecimal(int start, ArrowBuf buffer);
+ <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
</#if>
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 18fcac9..fe2b5c5 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -43,20 +43,42 @@ package org.apache.arrow.vector;
public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
- private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
private int allocationMonitor = 0;
- public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
+ <#if minor.class == "Decimal">
+
+ private int precision;
+ private int scale;
+
+ public ${minor.class}Vector(String name, BufferAllocator allocator, int precision, int scale) {
+ super(name, allocator);
+ this.precision = precision;
+ this.scale = scale;
+ }
+ <#else>
+ public ${minor.class}Vector(String name, BufferAllocator allocator) {
+ super(name, allocator);
+ }
+ </#if>
+
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.${minor.class?upper_case};
+ }
+
+ @Override
+ public Field getField() {
+ throw new UnsupportedOperationException("internal vector");
}
@Override
public FieldReader getReader(){
- return reader;
+ throw new UnsupportedOperationException("non-nullable vectors cannot be used in readers");
}
@Override
@@ -162,7 +184,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
}
- logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
+ logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", name, allocationSizeInBytes, newAllocationSize);
final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
newBuf.setBytes(0, data, 0, data.capacity());
final int halfNewCapacity = newBuf.capacity() / 2;
@@ -181,30 +203,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
data.setZero(0, data.capacity());
}
-// @Override
-// public void load(SerializedField metadata, ArrowBuf buffer) {
-// Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
-// final int actualLength = metadata.getBufferLength();
-// final int valueCount = metadata.getValueCount();
-// final int expectedLength = valueCount * ${type.width};
-// assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
-//
-// clear();
-// if (data != null) {
-// data.release(1);
-// }
-// data = buffer.slice(0, actualLength);
-// data.retain(1);
-// data.writerIndex(actualLength);
-// }
-
public TransferPair getTransferPair(BufferAllocator allocator){
- return new TransferImpl(getField(), allocator);
+ return new TransferImpl(name, allocator);
}
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator){
- return new TransferImpl(getField().withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -230,8 +235,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
private class TransferImpl implements TransferPair{
private ${minor.class}Vector to;
- public TransferImpl(MaterializedField field, BufferAllocator allocator){
- to = new ${minor.class}Vector(field, allocator);
+ public TransferImpl(String name, BufferAllocator allocator){
+ <#if minor.class == "Decimal">
+ to = new ${minor.class}Vector(name, allocator, precision, scale);
+ <#else>
+ to = new ${minor.class}Vector(name, allocator);
+ </#if>
}
public TransferImpl(${minor.class}Vector to) {
@@ -260,7 +269,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
}
public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
- <#if (type.width > 8)>
+ <#if (type.width > 8 || minor.class == "IntervalDay")>
from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
<#else> <#-- type.width <= 8 -->
data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
@@ -298,7 +307,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
return false;
}
- <#if (type.width > 8)>
+ <#if (type.width > 8 || minor.class == "IntervalDay")>
public ${minor.javaType!type.javaType} get(int index) {
return data.slice(index * ${type.width}, ${type.width});
@@ -416,31 +425,30 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
append(millis));
}
- <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+ <#elseif minor.class == "Decimal">
public void get(int index, ${minor.class}Holder holder) {
holder.start = index * ${type.width};
holder.buffer = data;
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
+ holder.scale = scale;
+ holder.precision = precision;
}
public void get(int index, Nullable${minor.class}Holder holder) {
holder.isSet = 1;
holder.start = index * ${type.width};
holder.buffer = data;
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
+ holder.scale = scale;
+ holder.precision = precision;
}
- @Override
- public ${friendlyType} getObject(int index) {
- <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
- // Get the BigDecimal object
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
- <#else>
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
- </#if>
+ @Override
+ public ${friendlyType} getObject(int index) {
+ byte[] bytes = new byte[${type.width}];
+ int start = ${type.width} * index;
+ data.getBytes(start, bytes, 0, ${type.width});
+ ${friendlyType} value = new BigDecimal(new BigInteger(bytes), scale);
+ return value;
}
<#else>
@@ -581,7 +589,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
* @param index position of the bit to set
* @param value value to set
*/
- <#if (type.width > 8)>
+ <#if (type.width > 8) || minor.class == "IntervalDay">
public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
data.setBytes(index * ${type.width}, value, 0, ${type.width});
}
@@ -653,7 +661,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
setSafe(index, holder.days, holder.milliseconds);
}
- <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
+ <#elseif minor.class == "Decimal">
public void set(int index, ${minor.class}Holder holder){
set(index, holder.start, holder.buffer);
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index 3005fca..1ed9287 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -19,9 +19,8 @@
<@pp.dropOutputFile />
<#list vv.types as type>
<#list type.minor as minor>
-<#list ["", "Nullable", "Repeated"] as holderMode>
+<#list ["", "Nullable"] as holderMode>
<#assign nullMode = holderMode />
-<#if holderMode == "Repeated"><#assign nullMode = "Nullable" /></#if>
<#assign lowerName = minor.class?uncap_first />
<#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
@@ -50,42 +49,18 @@ import org.joda.time.Period;
public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
private ${nullMode}${name}Holder holder;
-<#if holderMode == "Repeated" >
- private int index = -1;
- private ${holderMode}${name}Holder repeatedHolder;
-</#if>
-
public ${holderMode}${name}HolderReaderImpl(${holderMode}${name}Holder holder) {
-<#if holderMode == "Repeated" >
- this.holder = new ${nullMode}${name}Holder();
- this.repeatedHolder = holder;
-<#else>
this.holder = holder;
-</#if>
}
@Override
public int size() {
-<#if holderMode == "Repeated">
- return repeatedHolder.end - repeatedHolder.start;
-<#else>
throw new UnsupportedOperationException("You can't call size on a Holder value reader.");
-</#if>
}
@Override
public boolean next() {
-<#if holderMode == "Repeated">
- if(index + 1 < repeatedHolder.end) {
- index++;
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
- return true;
- } else {
- return false;
- }
-<#else>
throw new UnsupportedOperationException("You can't call next on a single value reader.");
-</#if>
}
@@ -95,19 +70,13 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
}
@Override
- public MajorType getType() {
-<#if holderMode == "Repeated">
- return this.repeatedHolder.TYPE;
-<#else>
- return this.holder.TYPE;
-</#if>
+ public MinorType getMinorType() {
+ return MinorType.${name?upper_case};
}
@Override
public boolean isSet() {
- <#if holderMode == "Repeated">
- return this.repeatedHolder.end!=this.repeatedHolder.start;
- <#elseif nullMode == "Nullable">
+ <#if holderMode == "Nullable">
return this.holder.isSet == 1;
<#else>
return true;
@@ -115,7 +84,6 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
}
-<#if holderMode != "Repeated">
@Override
public void read(${name}Holder h) {
<#list fields as field>
@@ -130,19 +98,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
</#list>
h.isSet = isSet() ? 1 : 0;
}
-</#if>
-<#if holderMode == "Repeated">
- @Override
- public ${friendlyType} read${safeType}(int index){
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + index, holder);
- ${friendlyType} value = read${safeType}();
- if (this.index > -1) {
- repeatedHolder.vector.getAccessor().get(repeatedHolder.start + this.index, holder);
- }
- return value;
- }
-</#if>
@Override
public ${friendlyType} read${safeType}(){
@@ -176,29 +132,10 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
Period p = new Period();
return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-<#elseif minor.class == "Decimal9" ||
- minor.class == "Decimal18" >
- BigInteger value = BigInteger.valueOf(holder.value);
- return new BigDecimal(value, holder.scale);
-
-<#elseif minor.class == "Decimal28Dense" ||
- minor.class == "Decimal38Dense">
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale,
- holder.maxPrecision,
- holder.WIDTH);
-
-<#elseif minor.class == "Decimal28Sparse" ||
- minor.class == "Decimal38Sparse">
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale);
-
<#elseif minor.class == "Bit" >
return new Boolean(holder.value != 0);
+<#elseif minor.class == "Decimal" >
+ return (BigDecimal) readSingleObject();
<#else>
${friendlyType} value = new ${friendlyType}(this.holder.value);
return value;
@@ -208,15 +145,7 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
@Override
public Object readObject() {
-<#if holderMode == "Repeated" >
- List<Object> valList = Lists.newArrayList();
- for (int i = repeatedHolder.start; i < repeatedHolder.end; i++) {
- valList.add(repeatedHolder.vector.getAccessor().getObject(i));
- }
- return valList;
-<#else>
return readSingleObject();
-</#if>
}
private Object readSingleObject() {
@@ -239,6 +168,9 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
Text text = new Text();
text.set(value);
return text;
+<#elseif minor.class == "Decimal" >
+ return new BigDecimal(new BigInteger(value), holder.scale);
+
</#if>
<#elseif minor.class == "Interval">
@@ -249,11 +181,6 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
Period p = new Period();
return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-<#elseif minor.class == "Decimal9" ||
- minor.class == "Decimal18" >
- BigInteger value = BigInteger.valueOf(holder.value);
- return new BigDecimal(value, holder.scale);
-
<#elseif minor.class == "Decimal28Dense" ||
minor.class == "Decimal38Dense">
return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
@@ -272,13 +199,18 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#elseif minor.class == "Bit" >
return new Boolean(holder.value != 0);
+<#elseif minor.class == "Decimal">
+ byte[] bytes = new byte[${type.width}];
+ holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
+ ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
+ return value;
<#else>
${friendlyType} value = new ${friendlyType}(this.holder.value);
return value;
</#if>
}
-<#if holderMode != "Repeated" && nullMode != "Nullable">
+<#if nullMode != "Nullable">
public void copyAsValue(${minor.class?cap_first}Writer writer){
writer.write(holder);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/codegen/templates/ListWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ListWriters.java b/java/vector/src/main/codegen/templates/ListWriters.java
deleted file mode 100644
index 94b812b..0000000
--- a/java/vector/src/main/codegen/templates/ListWriters.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-<@pp.dropOutputFile />
-
-<#list ["Single", "Repeated"] as mode>
-<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${mode}ListWriter.java" />
-
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.arrow.vector.complex.impl;
-<#if mode == "Single">
- <#assign containerClass = "AbstractContainerVector" />
- <#assign index = "idx()">
-<#else>
- <#assign containerClass = "RepeatedListVector" />
- <#assign index = "currentChildIndex">
-</#if>
-
-
-<#include "/@includes/vv_imports.ftl" />
-
-/*
- * This class is generated using FreeMarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-public class ${mode}ListWriter extends AbstractFieldWriter {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${mode}ListWriter.class);
-
- static enum Mode { INIT, IN_MAP, IN_LIST <#list vv.types as type><#list type.minor as minor>, IN_${minor.class?upper_case}</#list></#list> }
-
- private final String name;
- protected final ${containerClass} container;
- private Mode mode = Mode.INIT;
- private FieldWriter writer;
- protected RepeatedValueVector innerVector;
-
- <#if mode == "Repeated">private int currentChildIndex = 0;</#if>
- public ${mode}ListWriter(String name, ${containerClass} container, FieldWriter parent){
- super(parent);
- this.name = name;
- this.container = container;
- }
-
- public ${mode}ListWriter(${containerClass} container, FieldWriter parent){
- super(parent);
- this.name = null;
- this.container = container;
- }
-
- @Override
- public void allocate() {
- if(writer != null) {
- writer.allocate();
- }
-
- <#if mode == "Repeated">
- container.allocateNew();
- </#if>
- }
-
- @Override
- public void clear() {
- if (writer != null) {
- writer.clear();
- }
- }
-
- @Override
- public void close() {
- clear();
- container.close();
- if (innerVector != null) {
- innerVector.close();
- }
- }
-
- @Override
- public int getValueCapacity() {
- return innerVector == null ? 0 : innerVector.getValueCapacity();
- }
-
- public void setValueCount(int count){
- if(innerVector != null) innerVector.getMutator().setValueCount(count);
- }
-
- @Override
- public MapWriter map() {
- switch(mode) {
- case INIT:
- int vectorCount = container.size();
- final RepeatedMapVector vector = container.addOrGet(name, RepeatedMapVector.TYPE, RepeatedMapVector.class);
- innerVector = vector;
- writer = new RepeatedMapWriter(vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_MAP;
- return writer;
- case IN_MAP:
- return writer;
- }
-
- throw new RuntimeException(getUnsupportedErrorMsg("MAP", mode.name()));
-
- }
-
- @Override
- public ListWriter list() {
- switch(mode) {
- case INIT:
- final int vectorCount = container.size();
- final RepeatedListVector vector = container.addOrGet(name, RepeatedListVector.TYPE, RepeatedListVector.class);
- innerVector = vector;
- writer = new RepeatedListWriter(null, vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_LIST;
- return writer;
- case IN_LIST:
- return writer;
- }
-
- throw new RuntimeException(getUnsupportedErrorMsg("LIST", mode.name()));
-
- }
-
- <#list vv.types as type><#list type.minor as minor>
- <#assign lowerName = minor.class?uncap_first />
- <#assign upperName = minor.class?upper_case />
- <#assign capName = minor.class?cap_first />
- <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
-
- private static final MajorType ${upperName}_TYPE = Types.repeated(MinorType.${upperName});
-
- @Override
- public ${capName}Writer ${lowerName}() {
- switch(mode) {
- case INIT:
- final int vectorCount = container.size();
- final Repeated${capName}Vector vector = container.addOrGet(name, ${upperName}_TYPE, Repeated${capName}Vector.class);
- innerVector = vector;
- writer = new Repeated${capName}WriterImpl(vector, this);
- if(vectorCount != container.size()) {
- writer.allocate();
- }
- writer.setPosition(${index});
- mode = Mode.IN_${upperName};
- return writer;
- case IN_${upperName}:
- return writer;
- }
-
- throw new RuntimeException(getUnsupportedErrorMsg("${upperName}", mode.name()));
-
- }
- </#list></#list>
-
- public MaterializedField getField() {
- return container.getField();
- }
-
- <#if mode == "Repeated">
-
- public void startList() {
- final RepeatedListVector list = (RepeatedListVector) container;
- final RepeatedListVector.RepeatedMutator mutator = list.getMutator();
-
- // make sure that the current vector can support the end position of this list.
- if(container.getValueCapacity() <= idx()) {
- mutator.setValueCount(idx()+1);
- }
-
- // update the repeated vector to state that there is current+1 objects.
- final RepeatedListHolder h = new RepeatedListHolder();
- list.getAccessor().get(idx(), h);
- if (h.start >= h.end) {
- mutator.startNewValue(idx());
- }
- currentChildIndex = container.getMutator().add(idx());
- if(writer != null) {
- writer.setPosition(currentChildIndex);
- }
- }
-
- public void endList() {
- // noop, we initialize state at start rather than end.
- }
- <#else>
-
- public void setPosition(int index) {
- super.setPosition(index);
- if(writer != null) {
- writer.setPosition(index);
- }
- }
-
- public void startList() {
- // noop
- }
-
- public void endList() {
- // noop
- }
- </#if>
-
- private String getUnsupportedErrorMsg(String expected, String found) {
- final String f = found.substring(3);
- return String.format("In a list of type %s, encountered a value of type %s. "+
- "Arrow does not support lists of different types.",
- f, expected
- );
- }
-}
-</#list>