You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2009/02/06 02:34:53 UTC
svn commit: r741366 - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/udf/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: zshao
Date: Fri Feb 6 01:34:52 2009
New Revision: 741366
URL: http://svn.apache.org/viewvc?rev=741366&view=rev
Log:
HIVE-269. Add log/exp/pow UDF functions to Hive. (zshao)
Added:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLn.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java
hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf7.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/udf7.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=741366&r1=741365&r2=741366&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Feb 6 01:34:52 2009
@@ -26,6 +26,8 @@
NEW FEATURES
+ HIVE-269. Add log/exp/pow UDF functions to Hive. (zshao)
+
HIVE-258. New UDF IF(test, valueTrue, valueFalseOrNull). (zshao)
HIVE-105. Automatically estimate number of required reducers. (zshao)
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=741366&r1=741365&r2=741366&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Fri Feb 6 01:34:52 2009
@@ -58,6 +58,14 @@
registerUDF("ceil", UDFCeil.class, OperatorType.PREFIX, false);
registerUDF("ceiling", UDFCeil.class, OperatorType.PREFIX, false);
registerUDF("rand", UDFRand.class, OperatorType.PREFIX, false);
+
+ registerUDF("ln", UDFLn.class, OperatorType.PREFIX, false);
+ registerUDF("log2", UDFLog2.class, OperatorType.PREFIX, false);
+ registerUDF("log10", UDFLog10.class, OperatorType.PREFIX, false);
+ registerUDF("log", UDFLog.class, OperatorType.PREFIX, false);
+ registerUDF("exp", UDFExp.class, OperatorType.PREFIX, false);
+ registerUDF("power", UDFPower.class, OperatorType.PREFIX, false);
+ registerUDF("pow", UDFPower.class, OperatorType.PREFIX, false);
registerUDF("upper", UDFUpper.class, OperatorType.PREFIX, false);
registerUDF("lower", UDFLower.class, OperatorType.PREFIX, false);
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFExp.java Fri Feb 6 01:34:52 2009
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFExp extends UDF {
+
+ private static Log LOG = LogFactory.getLog(UDFExp.class.getName());
+
+ public UDFExp() {
+ }
+
+ /**
+ * Raise e (the base of natural logarithm) to the power of a.
+ */
+ public Double evaluate(Double a) {
+ if (a == null) {
+ return null;
+ } else {
+ return Double.valueOf(Math.exp(a));
+ }
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLn.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLn.java?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLn.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLn.java Fri Feb 6 01:34:52 2009
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFLn extends UDF {
+
+ private static Log LOG = LogFactory.getLog(UDFLn.class.getName());
+
+ public UDFLn() {
+ }
+
+ /**
+ * Returns the natural logarithm of "a".
+ */
+ public Double evaluate(Double a) {
+ if (a == null || a <= 0.0) {
+ return null;
+ } else {
+ return Double.valueOf(Math.log(a));
+ }
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java Fri Feb 6 01:34:52 2009
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFLog extends UDF {
+
+ private static Log LOG = LogFactory.getLog(UDFLog.class.getName());
+
+ public UDFLog() {
+ }
+
+ /**
+ * Returns the natural logarithm of "a".
+ */
+ public Double evaluate(Double a) {
+ if (a == null || a <= 0.0) {
+ return null;
+ } else {
+ return Double.valueOf(Math.log(a));
+ }
+ }
+
+ /**
+ * Returns the logarithm of "a" with base "base".
+ */
+ public Double evaluate(Double base, Double a) {
+ if (a == null || a <= 0.0 || base == null || base <= 1.0) {
+ return null;
+ } else {
+ return Double.valueOf(Math.log(a)/Math.log(base));
+ }
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog10.java Fri Feb 6 01:34:52 2009
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFLog10 extends UDF {
+
+ private static Log LOG = LogFactory.getLog(UDFLog10.class.getName());
+
+ private static double log10 = Math.log(10.0);
+ public UDFLog10() {
+ }
+
+ /**
+ * Returns the logarithm of "a" with base 10.
+ */
+ public Double evaluate(Double a) {
+ if (a == null || a <= 0.0) {
+ return null;
+ } else {
+ return Double.valueOf(Math.log(a)/log10);
+ }
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog2.java Fri Feb 6 01:34:52 2009
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFLog2 extends UDF {
+
+ private static Log LOG = LogFactory.getLog(UDFLog2.class.getName());
+
+ private static double log2 = Math.log(2.0);
+ public UDFLog2() {
+ }
+
+ /**
+ * Returns the logarithm of "a" with base 2.
+ */
+ public Double evaluate(Double a) {
+ if (a == null || a <= 0.0) {
+ return null;
+ } else {
+ return Double.valueOf(Math.log(a)/log2);
+ }
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFPower.java Fri Feb 6 01:34:52 2009
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+
+public class UDFPower extends UDF {
+
+ private static Log LOG = LogFactory.getLog(UDFPower.class.getName());
+
+ public UDFPower() {
+ }
+
+ /**
+ * Raise a to the power of b.
+ */
+ public Double evaluate(Double a, Double b) {
+ if (a == null || b == null) {
+ return null;
+ } else {
+ return Double.valueOf(Math.pow(a, b));
+ }
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf7.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf7.q?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf7.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/udf7.q Fri Feb 6 01:34:52 2009
@@ -0,0 +1,17 @@
+CREATE TABLE dest1(c1 STRING) STORED AS TEXTFILE;
+
+FROM src INSERT OVERWRITE TABLE dest1 SELECT ' abc ' WHERE src.key = 86;
+
+EXPLAIN
+SELECT LN(3.0), LN(0.0), LN(-1), LOG(3.0), LOG(0.0), LOG(-1), LOG2(3.0),
+ LOG2(0.0), LOG2(-1), LOG10(3.0), LOG10(0.0), LOG10(-1), LOG(2, 3.0),
+ LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), EXP(2.0),
+ POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5),
+ POWER(-1, 0.5), POWER(-1, 2) FROM dest1;
+
+SELECT LN(3.0), LN(0.0), LN(-1), LOG(3.0), LOG(0.0), LOG(-1), LOG2(3.0),
+ LOG2(0.0), LOG2(-1), LOG10(3.0), LOG10(0.0), LOG10(-1), LOG(2, 3.0),
+ LOG(2, 0.0), LOG(2, -1), LOG(0.5, 2), LOG(2, 0.5), EXP(2.0),
+ POW(2,3), POWER(2,3), POWER(2,-3), POWER(0.5, -3), POWER(4, 0.5),
+ POWER(-1, 0.5), POWER(-1, 2) FROM dest1;
+
Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/udf7.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/udf7.q.out?rev=741366&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/udf7.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/udf7.q.out Fri Feb 6 01:34:52 2009
@@ -0,0 +1,77 @@
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF dest1)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION LN 3.0)) (TOK_SELEXPR (TOK_FUNCTION LN 0.0)) (TOK_SELEXPR (TOK_FUNCTION LN (- 1))) (TOK_SELEXPR (TOK_FUNCTION LOG 3.0)) (TOK_SELEXPR (TOK_FUNCTION LOG 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG (- 1))) (TOK_SELEXPR (TOK_FUNCTION LOG2 3.0)) (TOK_SELEXPR (TOK_FUNCTION LOG2 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG2 (- 1))) (TOK_SELEXPR (TOK_FUNCTION LOG10 3.0)) (TOK_SELEXPR (TOK_FUNCTION LOG10 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG10 (- 1))) (TOK_SELEXPR (TOK_FUNCTION LOG 2 3.0)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 0.0)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 (- 1))) (TOK_SELEXPR (TOK_FUNCTION LOG 0.5 2)) (TOK_SELEXPR (TOK_FUNCTION LOG 2 0.5)) (TOK_SELEXPR (TOK_FUNCTION EXP 2.0)) (TOK_SELEXPR (TOK_FUNCTION POW 2 3)) (TOK_SELEXPR (TOK_FUNCTION POWER 2 3)) (TOK_SELEXPR (TOK_FUNCTION POWER 2 (- 3))) (TOK_SELEXPR (TOK_FUNCTION POWER 0.5 (- 3))) (TOK_SELEXPR (TOK_FUNCTIO
N POWER 4 0.5)) (TOK_SELEXPR (TOK_FUNCTION POWER (- 1) 0.5)) (TOK_SELEXPR (TOK_FUNCTION POWER (- 1) 2)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ dest1
+ Select Operator
+ Select Operator
+ expressions:
+ expr: ln(3.0)
+ type: double
+ expr: ln(0.0)
+ type: double
+ expr: ln(UDFToDouble(- 1))
+ type: double
+ expr: log(3.0)
+ type: double
+ expr: log(0.0)
+ type: double
+ expr: log(UDFToDouble(- 1))
+ type: double
+ expr: log2(3.0)
+ type: double
+ expr: log2(0.0)
+ type: double
+ expr: log2(UDFToDouble(- 1))
+ type: double
+ expr: log10(3.0)
+ type: double
+ expr: log10(0.0)
+ type: double
+ expr: log10(UDFToDouble(- 1))
+ type: double
+ expr: log(UDFToDouble(2), 3.0)
+ type: double
+ expr: log(UDFToDouble(2), 0.0)
+ type: double
+ expr: log(UDFToDouble(2), UDFToDouble(- 1))
+ type: double
+ expr: log(0.5, UDFToDouble(2))
+ type: double
+ expr: log(UDFToDouble(2), 0.5)
+ type: double
+ expr: exp(2.0)
+ type: double
+ expr: pow(UDFToDouble(2), UDFToDouble(3))
+ type: double
+ expr: pow(UDFToDouble(2), UDFToDouble(3))
+ type: double
+ expr: pow(UDFToDouble(2), UDFToDouble(- 3))
+ type: double
+ expr: pow(0.5, UDFToDouble(- 3))
+ type: double
+ expr: pow(UDFToDouble(4), 0.5)
+ type: double
+ expr: pow(UDFToDouble(- 1), 0.5)
+ type: double
+ expr: pow(UDFToDouble(- 1), UDFToDouble(2))
+ type: double
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+1.0986122886681096 NULL NULL 1.0986122886681096 NULL NULL 1.584962500721156 NULL NULL 0.4771212547196623 NULL NULL 1.584962500721156 NULL NULL NULL -1.0 7.38905609893065 8.0 8.0 0.125 8.0 2.0 NaN 1.0