You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/06/20 22:25:06 UTC
[30/32] git commit: DRILL-1024: Move hive storage code out of
'exec/java-exec' into 'contrib/storage-hive' module.
DRILL-1024: Move hive storage code out of 'exec/java-exec' into 'contrib/storage-hive' module.
+ Create two modules in contrib/storage-hive
++ contrib/storage-hive/hive-exec-shade: creates shaded hive-exec.jar
++ contrib/storage-hive/core: contains Hive storage code (schema, record reader and functions)
+ Update TestHiveUDFs.java to use BaseTestQuery instead of SimpleRootExec
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/980dc87b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/980dc87b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/980dc87b
Branch: refs/heads/master
Commit: 980dc87b5c2293e3ed60f588dcef67a6382e3715
Parents: 026d51a
Author: vkorukanti <ve...@gmail.com>
Authored: Wed Jun 18 11:35:23 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Fri Jun 20 10:56:16 2014 -0700
----------------------------------------------------------------------
contrib/storage-hive/core/pom.xml | 128 +++++
.../core/src/main/codegen/config.fmpp | 22 +
.../core/src/main/codegen/data/HiveTypes.tdd | 100 ++++
.../core/src/main/codegen/includes/license.ftl | 18 +
.../templates/ObjectInspectorHelper.java | 192 +++++++
.../codegen/templates/ObjectInspectors.java | 114 ++++
.../drill/exec/expr/HiveFuncHolderExpr.java | 76 +++
.../drill/exec/expr/fn/HiveFuncHolder.java | 272 ++++++++++
.../exec/expr/fn/HiveFunctionRegistry.java | 165 ++++++
.../hive/AbstractPrimitiveObjectInspector.java | 70 +++
.../expr/fn/impl/hive/DrillDeferredObject.java | 34 ++
.../drill/exec/planner/sql/HiveUDFOperator.java | 70 +++
.../exec/store/hive/HiveFieldConverter.java | 200 +++++++
.../drill/exec/store/hive/HiveReadEntry.java | 72 +++
.../drill/exec/store/hive/HiveRecordReader.java | 542 +++++++++++++++++++
.../apache/drill/exec/store/hive/HiveScan.java | 296 ++++++++++
.../exec/store/hive/HiveScanBatchCreator.java | 72 +++
.../exec/store/hive/HiveStoragePlugin.java | 87 +++
.../store/hive/HiveStoragePluginConfig.java | 74 +++
.../drill/exec/store/hive/HiveSubScan.java | 145 +++++
.../apache/drill/exec/store/hive/HiveTable.java | 331 +++++++++++
.../exec/store/hive/HiveTextRecordReader.java | 176 ++++++
.../exec/store/hive/schema/DrillHiveTable.java | 181 +++++++
.../store/hive/schema/DrillHiveViewTable.java | 40 ++
.../store/hive/schema/HiveDatabaseSchema.java | 63 +++
.../store/hive/schema/HiveSchemaFactory.java | 292 ++++++++++
.../resources/bootstrap-storage-plugins.json | 4 +
.../core/src/main/resources/drill-module.conf | 5 +
.../apache/drill/exec/fn/hive/TestHiveUDFs.java | 166 ++++++
.../exec/store/hive/HiveTestDataGenerator.java | 261 +++++++++
.../resources/functions/hive/GenericUDF.json | 45 ++
.../src/test/resources/functions/hive/UDF.json | 39 ++
.../core/src/test/resources/logback.xml | 64 +++
contrib/storage-hive/hive-exec-shade/pom.xml | 169 ++++++
.../src/main/resources/drill-module.conf | 5 +
contrib/storage-hive/pom.xml | 147 +----
.../src/main/resources/drill-module.conf | 5 -
distribution/pom.xml | 16 +-
distribution/src/assemble/bin.xml | 7 +-
exec/java-exec/pom.xml | 16 -
exec/java-exec/src/main/codegen/config.fmpp | 3 +-
.../src/main/codegen/data/HiveTypes.tdd | 100 ----
.../templates/ObjectInspectorHelper.java | 192 -------
.../codegen/templates/ObjectInspectors.java | 114 ----
.../drill/exec/expr/HiveFuncHolderExpr.java | 76 ---
.../drill/exec/expr/fn/HiveFuncHolder.java | 272 ----------
.../exec/expr/fn/HiveFunctionRegistry.java | 166 ------
.../hive/AbstractPrimitiveObjectInspector.java | 70 ---
.../expr/fn/impl/hive/DrillDeferredObject.java | 34 --
.../drill/exec/planner/sql/HiveUDFOperator.java | 70 ---
.../drill/exec/record/RecordBatchLoader.java | 2 -
.../apache/drill/exec/record/WritableBatch.java | 2 -
.../exec/store/hive/HiveFieldConverter.java | 200 -------
.../drill/exec/store/hive/HiveReadEntry.java | 72 ---
.../drill/exec/store/hive/HiveRecordReader.java | 542 -------------------
.../apache/drill/exec/store/hive/HiveScan.java | 296 ----------
.../exec/store/hive/HiveScanBatchCreator.java | 72 ---
.../exec/store/hive/HiveStoragePlugin.java | 87 ---
.../store/hive/HiveStoragePluginConfig.java | 74 ---
.../drill/exec/store/hive/HiveSubScan.java | 145 -----
.../apache/drill/exec/store/hive/HiveTable.java | 331 -----------
.../exec/store/hive/HiveTextRecordReader.java | 176 ------
.../exec/store/hive/schema/DrillHiveTable.java | 181 -------
.../store/hive/schema/DrillHiveViewTable.java | 40 --
.../store/hive/schema/HiveDatabaseSchema.java | 63 ---
.../store/hive/schema/HiveSchemaFactory.java | 292 ----------
.../drill/exec/physical/impl/TestHiveUDFs.java | 210 -------
.../exec/store/hive/HiveTestDataGenerator.java | 261 ---------
.../resources/functions/hive/GenericUDF.json | 45 --
.../src/test/resources/functions/hive/UDF.json | 39 --
exec/jdbc/pom.xml | 11 +
71 files changed, 4623 insertions(+), 4396 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
new file mode 100644
index 0000000..38e4a41
--- /dev/null
+++ b/contrib/storage-hive/core/pom.xml
@@ -0,0 +1,128 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+ xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.drill.contrib.storage-hive</groupId>
+ <artifactId>drill-contrib-storage-hive-parent</artifactId>
+ <version>1.0.0-m2-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>drill-storage-hive-core</artifactId>
+ <packaging>jar</packaging>
+ <name>contrib/hive-storage-plugin/core</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.exec</groupId>
+ <artifactId>drill-java-exec</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.contrib.storage-hive</groupId>
+ <artifactId>drill-hive-exec-shaded</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>net.hydromatic</groupId>
+ <artifactId>optiq-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill</groupId>
+ <artifactId>drill-common</artifactId>
+ <classifier>tests</classifier>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.drill.exec</groupId>
+ <artifactId>drill-java-exec</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.googlecode.fmpp-maven-plugin</groupId>
+ <artifactId>fmpp-maven-plugin</artifactId>
+ <version>1.0</version>
+ <configuration>
+ <cfgFile>src/main/codegen/config.fmpp</cfgFile>
+ <outputDirectory>target/generated-sources</outputDirectory>
+ <templateDirectory>src/main/codegen/templates</templateDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <id>generate-fmpp-sources</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.5</version>
+ <executions>
+ <execution>
+ <id>add-fmpp-sources</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test-jar</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/config.fmpp b/contrib/storage-hive/core/src/main/codegen/config.fmpp
new file mode 100644
index 0000000..cd36891
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/codegen/config.fmpp
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data: {
+ drillOI:tdd(../data/HiveTypes.tdd)
+}
+freemarkerLinks: {
+ includes: includes/
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
new file mode 100644
index 0000000..c23f981
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ map: [
+ {
+ hiveType: "BOOLEAN",
+ hiveOI: "BooleanObjectInspector",
+ serdeConstant: "BOOLEAN_TYPE_NAME",
+ javaType: "boolean",
+ minorType: "BIT",
+ holder: "Bit"
+ },
+ {
+ hiveType: "BYTE",
+ hiveOI: "ByteObjectInspector",
+ serdeConstant: "TINYINT_TYPE_NAME",
+ javaType: "byte",
+ minorType: "TINYINT",
+ holder: "TinyInt"
+ },
+ {
+ hiveType: "SHORT",
+ hiveOI: "ShortObjectInspector",
+ serdeConstant: "SMALLINT_TYPE_NAME",
+ javaType: "short",
+ minorType: "SMALLINT",
+ holder: "SmallInt"
+ },
+ {
+ hiveType: "INT",
+ hiveOI: "IntObjectInspector",
+ serdeConstant: "INT_TYPE_NAME",
+ javaType: "int",
+ minorType: "INT",
+ holder: "Int"
+ },
+ {
+ hiveType: "LONG",
+ hiveOI: "LongObjectInspector",
+ serdeConstant: "BIGINT_TYPE_NAME",
+ javaType: "long",
+ minorType: "BIGINT",
+ holder: "BigInt"
+ },
+ {
+ hiveType: "FLOAT",
+ hiveOI: "FloatObjectInspector",
+ serdeConstant: "FLOAT_TYPE_NAME",
+ javaType: "float",
+ minorType: "FLOAT4",
+ holder: "Float4"
+ },
+ {
+ hiveType: "DOUBLE",
+ hiveOI: "DoubleObjectInspector",
+ serdeConstant: "DOUBLE_TYPE_NAME",
+ javaType: "double",
+ minorType: "FLOAT8",
+ holder: "Float8"
+ },
+ {
+ hiveType: "VARCHAR",
+ hiveOI: "HiveVarcharObjectInspector",
+ serdeConstant: "VARCHAR_TYPE_NAME",
+ javaType: "",
+ minorType: "VARCHAR",
+ holder: "VarChar"
+ },
+ {
+ hiveType: "STRING",
+ hiveOI: "StringObjectInspector",
+ serdeConstant: "STRING_TYPE_NAME",
+ javaType: "",
+ minorType: "VAR16CHAR",
+ holder: "Var16Char"
+ },
+ {
+ hiveType: "BINARY",
+ hiveOI: "BinaryObjectInspector",
+ serdeConstant: "BINARY_TYPE_NAME",
+ javaType: "",
+ minorType: "VARBINARY",
+ holder: "VarBinary"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/codegen/includes/license.ftl
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/includes/license.ftl b/contrib/storage-hive/core/src/main/codegen/includes/license.ftl
new file mode 100644
index 0000000..0455fd8
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/codegen/includes/license.ftl
@@ -0,0 +1,18 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
new file mode 100644
index 0000000..22a9eb2
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/ObjectInspectorHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import com.sun.codemodel.*;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.DirectExpression;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ObjectInspectorHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ObjectInspectorHelper.class);
+
+ private static Map<MinorType, Class> OIMAP = new HashMap<>();
+ static {
+<#list drillOI.map as entry>
+ OIMAP.put(MinorType.${entry.minorType}, Drill${entry.holder}ObjectInspector.class);
+</#list>
+ }
+
+ public static ObjectInspector getDrillObjectInspector(MinorType drillType) {
+ if (OIMAP.containsKey(drillType)) {
+ try {
+ return (ObjectInspector)OIMAP.get(drillType).newInstance();
+ } catch(InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Failed to instantiate ObjectInspector", e);
+ }
+ }
+
+ throw new UnsupportedOperationException(drillType.toString());
+ }
+
+ public static JBlock initReturnValueHolder(JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) {
+ JBlock block = new JBlock(false, false);
+ switch(oi.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
+ switch(poi.getPrimitiveCategory()) {
+<#list drillOI.map as entry>
+ case ${entry.hiveType}:{
+ JType holderClass = TypeHelper.getHolderType(m, returnType, TypeProtos.DataMode.OPTIONAL);
+ block.assign(returnValueHolder, JExpr._new(holderClass));
+
+ <#if entry.hiveType == "VARCHAR" || entry.hiveType == "STRING" || entry.hiveType == "BINARY">
+ block.assign(returnValueHolder.ref("buffer"),
+ m.directClass(io.netty.buffer.Unpooled.class.getCanonicalName())
+ .staticInvoke("wrappedBuffer")
+ .arg(JExpr.newArray(m._ref(byte.class), JExpr.lit(1000))));
+ </#if>
+ return block;
+ }
+</#list>
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", poi.getPrimitiveCategory().toString()));
+ }
+ }
+
+ case MAP:
+ case LIST:
+ case STRUCT:
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", oi.getCategory().toString()));
+ }
+ }
+
+ private static Map<PrimitiveCategory, MinorType> TYPE_HIVE2DRILL = new HashMap<>();
+ static {
+<#list drillOI.map as entry>
+ TYPE_HIVE2DRILL.put(PrimitiveCategory.${entry.hiveType}, MinorType.${entry.minorType});
+</#list>
+ }
+
+ public static MinorType getDrillType(ObjectInspector oi) {
+ switch(oi.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
+ if (TYPE_HIVE2DRILL.containsKey(poi.getPrimitiveCategory())) {
+ return TYPE_HIVE2DRILL.get(poi.getPrimitiveCategory());
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ case MAP:
+ case LIST:
+ case STRUCT:
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static JBlock getDrillObject(JCodeModel m, ObjectInspector oi,
+ JVar returnOI, JVar returnValueHolder, JVar returnValue) {
+ JBlock block = new JBlock(false, false);
+ switch(oi.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
+ switch(poi.getPrimitiveCategory()) {
+<#list drillOI.map as entry>
+ case ${entry.hiveType}:{
+ JConditional jc = block._if(returnValue.eq(JExpr._null()));
+ jc._then().assign(returnValueHolder.ref("isSet"), JExpr.lit(0));
+ jc._else().assign(returnValueHolder.ref("isSet"), JExpr.lit(1));
+ JVar castedOI = jc._else().decl(
+ m.directClass(${entry.hiveOI}.class.getCanonicalName()), "castOI", JExpr._null());
+ jc._else().assign(castedOI,
+ JExpr.cast(m.directClass(${entry.hiveOI}.class.getCanonicalName()), returnOI));
+
+ <#if entry.hiveType == "BOOLEAN">
+ JConditional booleanJC = jc._else()._if(castedOI.invoke("get").arg(returnValue));
+ booleanJC._then().assign(returnValueHolder.ref("value"), JExpr.lit(1));
+ booleanJC._else().assign(returnValueHolder.ref("value"), JExpr.lit(0));
+
+ <#elseif entry.hiveType == "VARCHAR">
+ JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data",
+ castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)
+ .invoke("getValue")
+ .invoke("getBytes"));
+
+ jc._else().add(returnValueHolder.ref("buffer")
+ .invoke("setBytes").arg(JExpr.lit(0)).arg(data));
+
+
+ jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));
+ jc._else().assign(returnValueHolder.ref("end"), data.ref("length"));
+
+ <#elseif entry.hiveType == "STRING">
+ JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data",
+ castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)
+ .invoke("getBytes").arg(DirectExpression.direct("com.google.common.base.Charsets.UTF_16")));
+ jc._else().add(returnValueHolder.ref("buffer")
+ .invoke("setBytes").arg(JExpr.lit(0)).arg(data));
+ jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));
+ jc._else().assign(returnValueHolder.ref("end"), data.ref("length"));
+ <#elseif entry.hiveType == "BINARY">
+
+ JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data",
+ castedOI.invoke("getPrimitiveJavaObject").arg(returnValue));
+ jc._else().add(returnValueHolder.ref("buffer")
+ .invoke("setBytes").arg(JExpr.lit(0)).arg(data));
+ jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));
+ jc._else().assign(returnValueHolder.ref("end"), data.ref("length"));
+
+ <#else>
+ jc._else().assign(returnValueHolder.ref("value"),
+ castedOI.invoke("get").arg(returnValue));
+ </#if>
+ return block;
+ }
+
+</#list>
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", poi.getPrimitiveCategory().toString()));
+ }
+ }
+
+ case MAP:
+ case LIST:
+ case STRUCT:
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", oi.getCategory().toString()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
new file mode 100644
index 0000000..9a8c837
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+<#list drillOI.map as entry>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.holder}ObjectInspector.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
+import org.apache.hadoop.io.Text;
+
+public class Drill${entry.holder}ObjectInspector extends AbstractPrimitiveObjectInspector
+ implements ${entry.hiveOI} {
+
+ @Override
+ public String getTypeName() {
+ return serdeConstants.${entry.serdeConstant};
+ }
+
+<#if entry.minorType == "VARCHAR">
+ @Override
+ public HiveVarcharWritable getPrimitiveWritableObject(Object o) {
+ HiveVarcharWritable valW = new HiveVarcharWritable();
+ valW.set(getPrimitiveJavaObject(o));
+ return valW;
+ }
+
+ @Override
+ public HiveVarchar getPrimitiveJavaObject(Object o) {
+ String val = ((VarCharHolder)o).toString();
+ return new HiveVarchar(val, HiveVarchar.MAX_VARCHAR_LENGTH);
+ }
+<#elseif entry.minorType == "VAR16CHAR">
+@Override
+ public Text getPrimitiveWritableObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getPrimitiveJavaObject(Object o) {
+ if (o instanceof Var16CharHolder)
+ return ((Var16CharHolder)o).toString();
+ else
+ return ((NullableVar16CharHolder)o).toString();
+ }
+<#elseif entry.minorType == "VARBINARY">
+@Override
+public org.apache.hadoop.io.BytesWritable getPrimitiveWritableObject(Object o) {
+ throw new UnsupportedOperationException();
+}
+
+@Override
+public byte[] getPrimitiveJavaObject(Object o) {
+ if (o instanceof VarBinaryHolder){
+ VarBinaryHolder h = (VarBinaryHolder)o;
+ byte[] buf = new byte[h.end-h.start];
+ h.buffer.getBytes(h.start, buf, 0, h.end-h.start);
+ return buf;
+ }else{
+ NullableVarBinaryHolder h = (NullableVarBinaryHolder)o;
+ byte[] buf = new byte[h.end-h.start];
+ h.buffer.getBytes(h.start, buf, 0, h.end-h.start);
+ return buf;
+
+ }
+}
+<#elseif entry.minorType == "BIT">
+ @Override
+ public boolean get(Object o) {
+ if (o instanceof BitHolder)
+ return ((BitHolder)o).value == 0 ? false : true;
+ else
+ return ((NullableBitHolder)o).value == 0 ? false : true;
+ }
+<#else>
+ @Override
+ public ${entry.javaType} get(Object o) {
+ if (o instanceof ${entry.holder}Holder)
+ return ((${entry.holder}Holder)o).value;
+ else
+ return ((Nullable${entry.holder}Holder)o).value;
+ }
+</#if>
+
+ @Override
+ public PrimitiveCategory getPrimitiveCategory() {
+ return PrimitiveCategory.${entry.hiveType};
+ }
+}
+
+</#list>
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java
new file mode 100644
index 0000000..fd19e3d
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.fn.FuncHolder;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import org.apache.drill.exec.expr.fn.HiveFuncHolder;
+
+public class HiveFuncHolderExpr extends FunctionHolderExpression implements Iterable<LogicalExpression>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFuncHolderExpr.class);
+ private HiveFuncHolder holder;
+
+ public HiveFuncHolderExpr(String nameUsed, HiveFuncHolder holder, List<LogicalExpression> args, ExpressionPosition pos) {
+ super(nameUsed, pos, args);
+ this.holder = holder;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return holder.getReturnType();
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return args.iterator();
+ }
+
+ public FuncHolder getHolder() {
+ return holder;
+ }
+
+ @Override
+ public boolean isAggregating() {
+ return holder.isAggregating();
+ }
+
+ @Override
+ public boolean argConstantOnly(int i) {
+ // looks like hive UDF has no notion of constant argument input
+ return false;
+ }
+
+ @Override
+ public boolean isRandom() {
+ return holder.isRandom();
+ }
+
+ @Override
+ public HiveFuncHolderExpr copy(List<LogicalExpression> args) {
+ return new HiveFuncHolderExpr(this.nameUsed, this.holder, args, this.getPosition());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
new file mode 100644
index 0000000..813d4c5
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JCatchBlock;
+import com.sun.codemodel.JClass;
+import com.sun.codemodel.JCodeModel;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JTryBlock;
+import com.sun.codemodel.JVar;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.HiveFuncHolderExpr;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.fn.impl.hive.DrillDeferredObject;
+import org.apache.drill.exec.expr.fn.impl.hive.ObjectInspectorHelper;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+
+import java.util.List;
+
+public class HiveFuncHolder extends AbstractFuncHolder {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class);
+
+ private MajorType[] argTypes;
+ private ObjectInspector returnOI;
+ private MajorType returnType;
+ private Class<? extends GenericUDF> genericUdfClazz;
+ private boolean isGenericUDF = true;
+ private Class<? extends UDF> udfClazz = null;
+ private String udfName = "";
+ private boolean isRandom;
+
+
+ /**
+ * Create holder for GenericUDF
+ * @param genericUdfClazz implementation class
+ * @param argTypes
+ * @param returnOI
+ * @param returnType
+ */
+ public HiveFuncHolder(Class<? extends GenericUDF> genericUdfClazz, MajorType[] argTypes,
+ ObjectInspector returnOI, MajorType returnType, boolean isRandom) {
+ this.genericUdfClazz = genericUdfClazz;
+ this.argTypes = argTypes;
+ this.returnOI = returnOI;
+ this.returnType = returnType;
+ this.isRandom = isRandom;
+ }
+
+ /**
+ * Create holder for UDF
+ * @param udfName name of the UDF class
+ * @param udfClazz UDF implementation class
+ * @param argTypes
+ * @param returnOI
+ * @param returnType
+ */
+ public HiveFuncHolder(String udfName, Class< ? extends UDF> udfClazz, MajorType[] argTypes,
+ ObjectInspector returnOI, MajorType returnType, boolean isRandom) {
+ this(GenericUDFBridge.class, argTypes, returnOI, returnType, isRandom);
+ this.isGenericUDF = false;
+ this.udfClazz = udfClazz;
+ this.udfName = udfName;
+ }
+
+ /**
+ * UDF return type
+ */
+ public MajorType getReturnType() {
+ return returnType;
+ }
+
+ /**
+ * Aggregate function
+ */
+ public boolean isAggregating() {
+ // currently only simple UDFS are supported
+ return false;
+ }
+
+ /**
+ * is the function non-deterministic?
+ */
+ public boolean isRandom() {
+ return isRandom;
+ }
+
+ /**
+ * Start generating code
+ * @return workspace variables
+ */
+ @Override
+ public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables){
+ JVar[] workspaceJVars = new JVar[5];
+
+ workspaceJVars[0] = g.declareClassField("returnOI", g.getModel()._ref(ObjectInspector.class));
+ workspaceJVars[1] = g.declareClassField("udfInstance", g.getModel()._ref(GenericUDF.class));
+ workspaceJVars[2] = g.declareClassField("deferredObjects", g.getModel()._ref(DrillDeferredObject[].class));
+ workspaceJVars[3] = g.declareClassField("arguments", g.getModel()._ref(DrillDeferredObject[].class));
+ workspaceJVars[4] = g.declareClassField("returnValueHolder",
+ TypeHelper.getHolderType(g.getModel(), returnType.getMinorType(), TypeProtos.DataMode.OPTIONAL));
+
+ return workspaceJVars;
+ }
+
+ /**
+ * Complete code generation
+ * @param g
+ * @param inputVariables
+ * @param workspaceJVars
+ * @return HoldingContainer for return value
+ */
+ @Override
+ public HoldingContainer renderEnd(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
+ generateSetup(g, workspaceJVars);
+ return generateEval(g, inputVariables, workspaceJVars);
+ }
+
+ private JInvocation getUDFInstance(JCodeModel m) {
+ if (isGenericUDF) {
+ return JExpr._new(m.directClass(genericUdfClazz.getCanonicalName()));
+ } else {
+ return JExpr._new(m.directClass(GenericUDFBridge.class.getCanonicalName()))
+ .arg(JExpr.lit(udfName))
+ .arg(JExpr.lit(false))
+ .arg(JExpr.dotclass(m.directClass(udfClazz.getCanonicalName())));
+ }
+ }
+
+ @Override
+ public FunctionHolderExpression getExpr(String name, List<LogicalExpression> args, ExpressionPosition pos) {
+ return new HiveFuncHolderExpr(name, this, args, pos);
+ }
+
+ private void generateSetup(ClassGenerator<?> g, JVar[] workspaceJVars) {
+ JCodeModel m = g.getModel();
+ JBlock sub = new JBlock(true, true);
+
+ // declare and instantiate argument ObjectInspector's
+ JVar oiArray = sub.decl(
+ m._ref(ObjectInspector[].class),
+ "argOIs",
+ JExpr.newArray(m._ref(ObjectInspector.class), argTypes.length));
+
+ JClass oih = m.directClass(ObjectInspectorHelper.class.getCanonicalName());
+ JClass mt = m.directClass(TypeProtos.MinorType.class.getCanonicalName());
+ for(int i=0; i<argTypes.length; i++) {
+ sub.assign(
+ oiArray.component(JExpr.lit(i)),
+ oih.staticInvoke("getDrillObjectInspector")
+ .arg(mt.staticInvoke("valueOf")
+ .arg(JExpr.lit(argTypes[i].getMinorType().getNumber()))));
+ }
+
+ // declare and instantiate DeferredObject array
+ sub.assign(workspaceJVars[2], JExpr.newArray(m._ref(DrillDeferredObject.class), argTypes.length));
+
+ for(int i=0; i<argTypes.length; i++) {
+ sub.assign(
+ workspaceJVars[2].component(JExpr.lit(i)),
+ JExpr._new(m.directClass(DrillDeferredObject.class.getCanonicalName())));
+ }
+
+ // declare empty array for argument deferred objects
+ sub.assign(workspaceJVars[3], JExpr.newArray(m._ref(DrillDeferredObject.class), argTypes.length));
+
+ // create new instance of the UDF class
+ sub.assign(workspaceJVars[1], getUDFInstance(m));
+
+ // create try..catch block to initialize the UDF instance with argument OIs
+ JTryBlock udfInitTry = sub._try();
+ udfInitTry.body().assign(
+ workspaceJVars[0],
+ workspaceJVars[1].invoke("initialize")
+ .arg(oiArray));
+
+ JCatchBlock udfInitCatch = udfInitTry._catch(m.directClass(Exception.class.getCanonicalName()));
+ JVar exVar = udfInitCatch.param("ex");
+ udfInitCatch.body()
+ ._throw(JExpr._new(m.directClass(RuntimeException.class.getCanonicalName()))
+ .arg(JExpr.lit(String.format("Failed to initialize GenericUDF"))).arg(exVar));
+
+ sub.add(ObjectInspectorHelper.initReturnValueHolder(m, workspaceJVars[4], returnOI, returnType.getMinorType()));
+
+ // now add it to the doSetup block in Generated class
+ JBlock setup = g.getBlock(ClassGenerator.BlockType.SETUP);
+ setup.directStatement(String.format("/** start %s for function %s **/ ",
+ ClassGenerator.BlockType.SETUP.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+
+ setup.add(sub);
+
+ setup.directStatement(String.format("/** end %s for function %s **/ ",
+ ClassGenerator.BlockType.SETUP.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+ }
+
+ private HoldingContainer generateEval(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
+
+ HoldingContainer out = g.declare(returnType);
+
+ JCodeModel m = g.getModel();
+ JBlock sub = new JBlock(true, true);
+
+ // initialize DeferredObject's. For an optional type, assign the value holder only if it is not null
+ for(int i=0; i<argTypes.length; i++) {
+ if (inputVariables[i].isOptional()) {
+ JBlock conditionalBlock = new JBlock(false, false);
+ JConditional jc = conditionalBlock._if(inputVariables[i].getIsSet().ne(JExpr.lit(0)));
+ jc._then().assign(workspaceJVars[3].component(JExpr.lit(i)), workspaceJVars[2].component(JExpr.lit(i)));
+ jc._then().assign(JExpr.ref(workspaceJVars[3].component(JExpr.lit(i)), "valueHolder"), inputVariables[i].getHolder());
+ jc._else().assign(workspaceJVars[3].component(JExpr.lit(i)), JExpr._null());
+ sub.add(conditionalBlock);
+ } else {
+ sub.assign(workspaceJVars[3].component(JExpr.lit(i)), workspaceJVars[2].component(JExpr.lit(i)));
+ sub.assign(JExpr.ref(workspaceJVars[3].component(JExpr.lit(i)), "valueHolder"), inputVariables[i].getHolder());
+ }
+ }
+
+ // declare generic object for storing return value from GenericUDF.evaluate
+ JVar retVal = sub.decl(m._ref(Object.class), "ret");
+
+ // create try..catch block to call the GenericUDF instance with given input
+ JTryBlock udfEvalTry = sub._try();
+ udfEvalTry.body().assign(retVal,
+ workspaceJVars[1].invoke("evaluate").arg(workspaceJVars[3]));
+
+ JCatchBlock udfEvalCatch = udfEvalTry._catch(m.directClass(Exception.class.getCanonicalName()));
+ JVar exVar = udfEvalCatch.param("ex");
+ udfEvalCatch.body()
+ ._throw(JExpr._new(m.directClass(RuntimeException.class.getCanonicalName()))
+ .arg(JExpr.lit(String.format("GenericUDF.evaluate method failed"))).arg(exVar));
+
+ // get the ValueHolder from retVal and return ObjectInspector
+ sub.add(ObjectInspectorHelper.getDrillObject(m, returnOI, workspaceJVars[0], workspaceJVars[4], retVal));
+ sub.assign(out.getHolder(), workspaceJVars[4]);
+
+ // now add it to the doEval block in Generated class
+ JBlock setup = g.getBlock(ClassGenerator.BlockType.EVAL);
+ setup.directStatement(String.format("/** start %s for function %s **/ ",
+ ClassGenerator.BlockType.EVAL.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+ setup.add(sub);
+ setup.directStatement(String.format("/** end %s for function %s **/ ",
+ ClassGenerator.BlockType.EVAL.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+
+ return out;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
new file mode 100644
index 0000000..b9369ed
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.google.common.collect.Sets;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.exec.expr.fn.impl.hive.ObjectInspectorHelper;
+import org.apache.drill.exec.planner.sql.DrillOperatorTable;
+import org.apache.drill.exec.planner.sql.HiveUDFOperator;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import com.google.common.collect.ArrayListMultimap;
+
+public class HiveFunctionRegistry implements PluggableFunctionRegistry{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveFunctionRegistry.class);
+
+ private ArrayListMultimap<String, Class<? extends GenericUDF>> methodsGenericUDF = ArrayListMultimap.create();
+ private ArrayListMultimap<String, Class<? extends UDF>> methodsUDF = ArrayListMultimap.create();
+ private HashSet<Class<?>> nonDeterministicUDFs = new HashSet<>();
+
+ /**
+ * Scan the classpath for implementation of GenericUDF/UDF interfaces,
+ * extracts function annotation and store the
+ * (function name) --> (implementation class) mappings.
+ * @param config
+ */
+ public HiveFunctionRegistry(DrillConfig config){
+ Set<Class<? extends GenericUDF>> genericUDFClasses = PathScanner.scanForImplementations(GenericUDF.class, null);
+ for (Class<? extends GenericUDF> clazz : genericUDFClasses)
+ register(clazz, methodsGenericUDF);
+
+ Set<Class<? extends UDF>> udfClasses = PathScanner.scanForImplementations(UDF.class, null);
+ for (Class<? extends UDF> clazz : udfClasses)
+ register(clazz, methodsUDF);
+ }
+
+ @Override
+ public void register(DrillOperatorTable operatorTable) {
+ for (String name : Sets.union(methodsGenericUDF.asMap().keySet(), methodsUDF.asMap().keySet())) {
+ operatorTable.add(name, new HiveUDFOperator(name.toUpperCase()));
+ }
+ }
+
+ private <C,I> void register(Class<? extends I> clazz, ArrayListMultimap<String,Class<? extends I>> methods) {
+ Description desc = clazz.getAnnotation(Description.class);
+ String[] names;
+ if(desc != null){
+ names = desc.name().split(",");
+ for(int i=0; i<names.length; i++) names[i] = names[i].trim();
+ }else{
+ names = new String[]{clazz.getName().replace('.', '_')};
+ }
+
+ UDFType type = clazz.getAnnotation(UDFType.class);
+ if (type != null && type.deterministic()) nonDeterministicUDFs.add(clazz);
+
+
+ for(int i=0; i<names.length;i++){
+ methods.put(names[i].toLowerCase(), clazz);
+ }
+ }
+
+ /**
+ * Find the UDF class for given function name and check if it accepts the given input argument
+ * types. If a match is found, create a holder and return
+ * @param call
+ * @return
+ */
+ @Override
+ public HiveFuncHolder getFunction(FunctionCall call){
+ HiveFuncHolder holder;
+ MajorType[] argTypes = new MajorType[call.args.size()];
+ ObjectInspector[] argOIs = new ObjectInspector[call.args.size()];
+ for(int i=0; i<call.args.size(); i++) {
+ argTypes[i] = call.args.get(i).getMajorType();
+ argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMinorType());
+ }
+
+ String funcName = call.getName().toLowerCase();
+
+ // search in GenericUDF list
+ for(Class<? extends GenericUDF> clazz: methodsGenericUDF.get(funcName)) {
+ holder = matchAndCreateGenericUDFHolder(clazz, argTypes, argOIs);
+ if(holder != null)
+ return holder;
+ }
+
+ // search in UDF list
+ for (Class<? extends UDF> clazz : methodsUDF.get(funcName)) {
+ holder = matchAndCreateUDFHolder(call.getName(), clazz, argTypes, argOIs);
+ if (holder != null)
+ return holder;
+ }
+
+ return null;
+ }
+
+ private HiveFuncHolder matchAndCreateGenericUDFHolder(Class<? extends GenericUDF> udfClazz,
+ MajorType[] argTypes,
+ ObjectInspector[] argOIs) {
+ // probe UDF to find if the arg types and acceptable
+ // if acceptable create a holder object
+ try {
+ GenericUDF udfInstance = udfClazz.newInstance();
+ ObjectInspector returnOI = udfInstance.initialize(argOIs);
+ return new HiveFuncHolder(
+ udfClazz,
+ argTypes,
+ returnOI,
+ Types.optional(ObjectInspectorHelper.getDrillType(returnOI)),
+ nonDeterministicUDFs.contains(udfClazz));
+ } catch(IllegalAccessException | InstantiationException e) {
+ logger.debug("Failed to instantiate class", e);
+ } catch(Exception e) { /*ignore this*/ }
+
+ return null;
+ }
+
+ private HiveFuncHolder matchAndCreateUDFHolder(String udfName,
+ Class<? extends UDF> udfClazz,
+ MajorType[] argTypes,
+ ObjectInspector[] argOIs) {
+ try {
+ GenericUDF udfInstance = new GenericUDFBridge(udfName, false/* is operator */, udfClazz);
+ ObjectInspector returnOI = udfInstance.initialize(argOIs);
+
+ return new HiveFuncHolder(
+ udfName,
+ udfClazz,
+ argTypes,
+ returnOI,
+ Types.optional(ObjectInspectorHelper.getDrillType(returnOI)),
+ nonDeterministicUDFs.contains(udfClazz));
+ } catch(Exception e) { /*ignore this*/ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
new file mode 100644
index 0000000..04b552e
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams;
+
+
+public abstract class AbstractPrimitiveObjectInspector implements PrimitiveObjectInspector {
+
+ @Override
+ public Class<?> getPrimitiveWritableClass() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getPrimitiveWritableObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Class<?> getJavaPrimitiveClass() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getPrimitiveJavaObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object copyObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean preferWritable() {
+ return false;
+ }
+
+ @Override
+ public BaseTypeParams getTypeParams() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setTypeParams(BaseTypeParams baseTypeParams) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.PRIMITIVE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java
new file mode 100644
index 0000000..fbc5f05
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import org.apache.drill.exec.expr.holders.ValueHolder;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+
+public class DrillDeferredObject implements GenericUDF.DeferredObject {
+ public ValueHolder valueHolder;
+
+ @Override
+ public void prepare(int version) throws HiveException {}
+
+ @Override
+ public Object get() throws HiveException {
+ return valueHolder;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java
new file mode 100644
index 0000000..71860c3
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/HiveUDFOperator.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.planner.sql;
+
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.sql.SqlCall;
+import org.eigenbase.sql.SqlCallBinding;
+import org.eigenbase.sql.SqlFunction;
+import org.eigenbase.sql.SqlFunctionCategory;
+import org.eigenbase.sql.SqlIdentifier;
+import org.eigenbase.sql.SqlOperandCountRange;
+import org.eigenbase.sql.SqlOperator;
+import org.eigenbase.sql.parser.SqlParserPos;
+import org.eigenbase.sql.type.SqlOperandCountRanges;
+import org.eigenbase.sql.type.SqlOperandTypeChecker;
+import org.eigenbase.sql.type.SqlTypeName;
+import org.eigenbase.sql.validate.SqlValidator;
+import org.eigenbase.sql.validate.SqlValidatorScope;
+
+public class HiveUDFOperator extends SqlFunction {
+
+ public HiveUDFOperator(String name) {
+ super(new SqlIdentifier(name, SqlParserPos.ZERO), DynamicReturnType.INSTANCE, null, new ArgChecker(), null,
+ SqlFunctionCategory.USER_DEFINED_FUNCTION);
+ }
+
+ @Override
+ public RelDataType deriveType(SqlValidator validator, SqlValidatorScope scope, SqlCall call) {
+ return validator.getTypeFactory().createSqlType(SqlTypeName.ANY);
+ }
+
+ /** Argument Checker for variable number of arguments */
+ public static class ArgChecker implements SqlOperandTypeChecker {
+
+ public static ArgChecker INSTANCE = new ArgChecker();
+
+ private SqlOperandCountRange range = SqlOperandCountRanges.any();
+
+ @Override
+ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFailure) {
+ return true;
+ }
+
+ @Override
+ public SqlOperandCountRange getOperandCountRange() {
+ return range;
+ }
+
+ @Override
+ public String getAllowedSignatures(SqlOperator op, String opName) {
+ return opName + "(HiveUDF - Opaque)";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
new file mode 100644
index 0000000..5095d90
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
@@ -0,0 +1,200 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import java.util.Map;
+
+import org.apache.drill.exec.vector.NullableBigIntVector;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableDateVector;
+import org.apache.drill.exec.vector.NullableFloat4Vector;
+import org.apache.drill.exec.vector.NullableFloat8Vector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableSmallIntVector;
+import org.apache.drill.exec.vector.NullableTimeStampVector;
+import org.apache.drill.exec.vector.NullableTinyIntVector;
+import org.apache.drill.exec.vector.NullableVarBinaryVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.Text;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import com.google.common.collect.Maps;
+
+public abstract class HiveFieldConverter {
+
+ public abstract boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex);
+
+ private static Map<PrimitiveCategory, Class< ? extends HiveFieldConverter>> primMap = Maps.newHashMap();
+
+ static {
+ primMap.put(PrimitiveCategory.BINARY, Binary.class);
+ primMap.put(PrimitiveCategory.BOOLEAN, Boolean.class);
+ primMap.put(PrimitiveCategory.BYTE, Byte.class);
+ primMap.put(PrimitiveCategory.DECIMAL, Decimal.class);
+ primMap.put(PrimitiveCategory.DOUBLE, Double.class);
+ primMap.put(PrimitiveCategory.FLOAT, Float.class);
+ primMap.put(PrimitiveCategory.INT, Int.class);
+ primMap.put(PrimitiveCategory.LONG, Long.class);
+ primMap.put(PrimitiveCategory.SHORT, Short.class);
+ primMap.put(PrimitiveCategory.STRING, String.class);
+ primMap.put(PrimitiveCategory.VARCHAR, VarChar.class);
+ primMap.put(PrimitiveCategory.TIMESTAMP, Timestamp.class);
+ primMap.put(PrimitiveCategory.DATE, Date.class);
+ }
+
+
+ public static HiveFieldConverter create(TypeInfo typeInfo) throws IllegalAccessException, InstantiationException {
+ switch (typeInfo.getCategory()) {
+ case PRIMITIVE:
+ final PrimitiveCategory pCat = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
+ Class< ? extends HiveFieldConverter> clazz = primMap.get(pCat);
+ if (clazz != null)
+ return clazz.newInstance();
+
+ HiveRecordReader.throwUnsupportedHiveDataTypeError(pCat.toString());
+ break;
+
+ case LIST:
+ case MAP:
+ case STRUCT:
+ case UNION:
+ default:
+ HiveRecordReader.throwUnsupportedHiveDataTypeError(typeInfo.getCategory().toString());
+ }
+
+ return null;
+ }
+
+ public static class Binary extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableVarBinaryVector) outputVV).getMutator().setSafe(outputIndex, value, 0, value.length);
+ }
+ }
+
+ public static class Boolean extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final boolean value = (boolean) ((BooleanObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableBitVector) outputVV).getMutator().setSafe(outputIndex, value ? 1 : 0);
+ }
+ }
+
+ public static class Byte extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final byte value = (byte) ((ByteObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableTinyIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Decimal extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final HiveDecimal value = ((HiveDecimalObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ final byte[] strBytes = value.toString().getBytes();
+ return ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, strBytes, 0, strBytes.length);
+ }
+ }
+
+ public static class Double extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final double value = (double) ((DoubleObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableFloat8Vector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Float extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final float value = (float) ((FloatObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableFloat4Vector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Int extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final int value = (int) ((IntObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Long extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final long value = (long) ((LongObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableBigIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class Short extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final short value = (short) ((ShortObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ return ((NullableSmallIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+ }
+ }
+
+ public static class String extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final Text value = ((StringObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
+ final byte[] valueBytes = value.getBytes();
+ final int len = value.getLength();
+ return ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, valueBytes, 0, len);
+ }
+ }
+
+ public static class VarChar extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final Text value = ((HiveVarcharObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue).getTextValue();
+ final byte[] valueBytes = value.getBytes();
+ final int valueLen = value.getLength();
+ return ((NullableVarCharVector) outputVV).getMutator().setSafe(outputIndex, valueBytes, 0, valueLen);
+ }
+ }
+
+ public static class Timestamp extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final java.sql.Timestamp value = ((TimestampObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ final DateTime ts = new DateTime(value.getTime()).withZoneRetainFields(DateTimeZone.UTC);
+ return ((NullableTimeStampVector) outputVV).getMutator().setSafe(outputIndex, ts.getMillis());
+ }
+ }
+
+ public static class Date extends HiveFieldConverter {
+ public boolean setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
+ final java.sql.Date value = ((DateObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+ final DateTime date = new DateTime(value.getTime()).withZoneRetainFields(DateTimeZone.UTC);
+ return ((NullableDateVector) outputVV).getMutator().setSafe(outputIndex, date.getMillis());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/980dc87b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
new file mode 100644
index 0000000..32f793e
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveReadEntry.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import java.util.List;
+
+import net.hydromatic.optiq.Schema.TableType;
+
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
+
+public class HiveReadEntry {
+
+ @JsonProperty("table")
+ public HiveTable table;
+ @JsonProperty("partitions")
+ public List<HiveTable.HivePartition> partitions;
+
+ @JsonIgnore
+ private List<Partition> partitionsUnwrapped = Lists.newArrayList();
+
+ @JsonCreator
+ public HiveReadEntry(@JsonProperty("table") HiveTable table, @JsonProperty("partitions") List<HiveTable.HivePartition> partitions) {
+ this.table = table;
+ this.partitions = partitions;
+ if (partitions != null) {
+ for(HiveTable.HivePartition part : partitions) {
+ partitionsUnwrapped.add(part.getPartition());
+ }
+ }
+ }
+
+ @JsonIgnore
+ public Table getTable() {
+ return table.getTable();
+ }
+
+ @JsonIgnore
+ public List<Partition> getPartitions() {
+ return partitionsUnwrapped;
+ }
+
+ @JsonIgnore
+ public TableType getJdbcTableType() {
+ if (table.getTable().getTableType().equals(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW.toString())) {
+ return TableType.VIEW;
+ }
+
+ return TableType.TABLE;
+ }
+}
+