You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/12/05 02:32:30 UTC
svn commit: r1547986 [1/5] - in /hive/branches/tez: ./
ant/src/org/apache/hadoop/hive/ant/ beeline/
beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/
beeline/src/test/org/apache/hive/beeline/src/test/ cli/ common/
common/src/java/or...
Author: thejas
Date: Thu Dec 5 01:32:27 2013
New Revision: 1547986
URL: http://svn.apache.org/r1547986
Log:
Merge latest trunk into branch (from correct dir) (Thejas Nair)
Added:
hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java
- copied unchanged from r1547983, hive/trunk/beeline/src/java/org/apache/hive/beeline/BooleanCompletor.java
hive/branches/tez/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt
- copied unchanged from r1547983, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideColumn.txt
hive/branches/tez/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalar.txt
- copied unchanged from r1547983, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/ColumnDivideScalar.txt
hive/branches/tez/ql/src/gen/vectorization/ExpressionTemplates/ScalarDivideColumn.txt
- copied unchanged from r1547983, hive/trunk/ql/src/gen/vectorization/ExpressionTemplates/ScalarDivideColumn.txt
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DoubleColumnInList.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DoubleColumnInList.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IDoubleInExpr.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IDoubleInExpr.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ILongInExpr.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/ILongInExpr.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IStringInExpr.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/IStringInExpr.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColumnInList.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColumnInList.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnInList.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StringColumnInList.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDayOfMonthString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFDayOfMonthString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFHourString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFHourString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFMinuteString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFMinuteString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFMonthString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFMonthString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFSecondString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFSecondString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFTimestampFieldString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFUnixTimeStampString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFWeekOfYearString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFWeekOfYearString.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFYearString.java
- copied unchanged from r1547983, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorUDFYearString.java
hive/branches/tez/ql/src/test/queries/clientpositive/char_serde.q
- copied unchanged from r1547983, hive/trunk/ql/src/test/queries/clientpositive/char_serde.q
hive/branches/tez/ql/src/test/queries/clientpositive/vectorization_div0.q
- copied unchanged from r1547983, hive/trunk/ql/src/test/queries/clientpositive/vectorization_div0.q
hive/branches/tez/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q
- copied unchanged from r1547983, hive/trunk/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q
hive/branches/tez/ql/src/test/results/clientpositive/char_serde.q.out
- copied unchanged from r1547983, hive/trunk/ql/src/test/results/clientpositive/char_serde.q.out
hive/branches/tez/ql/src/test/results/clientpositive/vectorization_div0.q.out
- copied unchanged from r1547983, hive/trunk/ql/src/test/results/clientpositive/vectorization_div0.q.out
hive/branches/tez/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
- copied unchanged from r1547983, hive/trunk/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
Modified:
hive/branches/tez/ (props changed)
hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
hive/branches/tez/beeline/pom.xml
hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLine.java
hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BufferedRows.java
hive/branches/tez/beeline/src/java/org/apache/hive/beeline/Rows.java
hive/branches/tez/beeline/src/main/resources/BeeLine.properties
hive/branches/tez/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
hive/branches/tez/cli/pom.xml
hive/branches/tez/common/pom.xml
hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/tez/contrib/pom.xml
hive/branches/tez/hbase-handler/pom.xml
hive/branches/tez/hcatalog/pom.xml
hive/branches/tez/hcatalog/storage-handlers/hbase/pom.xml
hive/branches/tez/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateTableDesc.java
hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
hive/branches/tez/hwi/pom.xml
hive/branches/tez/jdbc/pom.xml
hive/branches/tez/metastore/pom.xml
hive/branches/tez/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql
hive/branches/tez/packaging/pom.xml
hive/branches/tez/packaging/src/main/assembly/bin.xml
hive/branches/tez/pom.xml
hive/branches/tez/ql/pom.xml
hive/branches/tez/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDoubleColumnInList.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColumnInList.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongColumn.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongScalar.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarDivideLongColumn.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFWeekOfYear.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFYear.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPDivide.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMinus.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMod.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPMultiply.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPPlus.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPosMod.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToUnixTimeStamp.java
hive/branches/tez/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizationContext.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmeticExpressions.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorLogicalExpressions.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorScalarColArithmetic.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorStringExpressions.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTimestampExpressions.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFPower.java
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_filter.q
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_groupby.q
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_join.q
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_limit.q
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_part.q
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_select.q
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_table.q
hive/branches/tez/ql/src/test/queries/clientpositive/annotate_stats_union.q
hive/branches/tez/ql/src/test/queries/clientpositive/decimal_precision.q
hive/branches/tez/ql/src/test/queries/clientpositive/decimal_udf.q
hive/branches/tez/ql/src/test/queries/clientpositive/show_partitions.q
hive/branches/tez/ql/src/test/results/clientpositive/decimal_precision.q.out
hive/branches/tez/ql/src/test/results/clientpositive/decimal_udf.q.out
hive/branches/tez/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out
hive/branches/tez/ql/src/test/results/clientpositive/show_partitions.q.out
hive/branches/tez/ql/src/test/results/clientpositive/showparts.q.out
hive/branches/tez/ql/src/test/results/clientpositive/vectorization_5.q.out
hive/branches/tez/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
hive/branches/tez/ql/src/test/templates/TestCliDriver.vm
hive/branches/tez/ql/src/test/templates/TestNegativeCliDriver.vm
hive/branches/tez/ql/src/test/templates/TestParse.vm
hive/branches/tez/ql/src/test/templates/TestParseNegative.vm
hive/branches/tez/serde/pom.xml
hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java
hive/branches/tez/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java
hive/branches/tez/service/pom.xml
hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
hive/branches/tez/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java
Propchange: hive/branches/tez/
------------------------------------------------------------------------------
Merged /hive/trunk:r1546165-1547983
Modified: hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original)
+++ hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Thu Dec 5 01:32:27 2013
@@ -43,19 +43,16 @@ public class GenVectorCode extends Task
{"ColumnArithmeticScalar", "Add", "long", "double", "+"},
{"ColumnArithmeticScalar", "Subtract", "long", "double", "-"},
{"ColumnArithmeticScalar", "Multiply", "long", "double", "*"},
- {"ColumnArithmeticScalar", "Divide", "long", "double", "/"},
{"ColumnArithmeticScalar", "Modulo", "long", "double", "%"},
{"ColumnArithmeticScalar", "Add", "double", "long", "+"},
{"ColumnArithmeticScalar", "Subtract", "double", "long", "-"},
{"ColumnArithmeticScalar", "Multiply", "double", "long", "*"},
- {"ColumnArithmeticScalar", "Divide", "double", "long", "/"},
{"ColumnArithmeticScalar", "Modulo", "double", "long", "%"},
{"ColumnArithmeticScalar", "Add", "double", "double", "+"},
{"ColumnArithmeticScalar", "Subtract", "double", "double", "-"},
{"ColumnArithmeticScalar", "Multiply", "double", "double", "*"},
- {"ColumnArithmeticScalar", "Divide", "double", "double", "/"},
{"ColumnArithmeticScalar", "Modulo", "double", "double", "%"},
{"ScalarArithmeticColumn", "Add", "long", "long", "+"},
@@ -66,19 +63,16 @@ public class GenVectorCode extends Task
{"ScalarArithmeticColumn", "Add", "long", "double", "+"},
{"ScalarArithmeticColumn", "Subtract", "long", "double", "-"},
{"ScalarArithmeticColumn", "Multiply", "long", "double", "*"},
- {"ScalarArithmeticColumn", "Divide", "long", "double", "/"},
{"ScalarArithmeticColumn", "Modulo", "long", "double", "%"},
{"ScalarArithmeticColumn", "Add", "double", "long", "+"},
{"ScalarArithmeticColumn", "Subtract", "double", "long", "-"},
{"ScalarArithmeticColumn", "Multiply", "double", "long", "*"},
- {"ScalarArithmeticColumn", "Divide", "double", "long", "/"},
{"ScalarArithmeticColumn", "Modulo", "double", "long", "%"},
{"ScalarArithmeticColumn", "Add", "double", "double", "+"},
{"ScalarArithmeticColumn", "Subtract", "double", "double", "-"},
{"ScalarArithmeticColumn", "Multiply", "double", "double", "*"},
- {"ScalarArithmeticColumn", "Divide", "double", "double", "/"},
{"ScalarArithmeticColumn", "Modulo", "double", "double", "%"},
{"ColumnArithmeticColumn", "Add", "long", "long", "+"},
@@ -89,21 +83,28 @@ public class GenVectorCode extends Task
{"ColumnArithmeticColumn", "Add", "long", "double", "+"},
{"ColumnArithmeticColumn", "Subtract", "long", "double", "-"},
{"ColumnArithmeticColumn", "Multiply", "long", "double", "*"},
- {"ColumnArithmeticColumn", "Divide", "long", "double", "/"},
{"ColumnArithmeticColumn", "Modulo", "long", "double", "%"},
{"ColumnArithmeticColumn", "Add", "double", "long", "+"},
{"ColumnArithmeticColumn", "Subtract", "double", "long", "-"},
{"ColumnArithmeticColumn", "Multiply", "double", "long", "*"},
- {"ColumnArithmeticColumn", "Divide", "double", "long", "/"},
{"ColumnArithmeticColumn", "Modulo", "double", "long", "%"},
{"ColumnArithmeticColumn", "Add", "double", "double", "+"},
{"ColumnArithmeticColumn", "Subtract", "double", "double", "-"},
{"ColumnArithmeticColumn", "Multiply", "double", "double", "*"},
- {"ColumnArithmeticColumn", "Divide", "double", "double", "/"},
{"ColumnArithmeticColumn", "Modulo", "double", "double", "%"},
+ {"ColumnDivideScalar", "Divide", "long", "double", "/"},
+ {"ColumnDivideScalar", "Divide", "double", "long", "/"},
+ {"ColumnDivideScalar", "Divide", "double", "double", "/"},
+ {"ScalarDivideColumn", "Divide", "long", "double", "/"},
+ {"ScalarDivideColumn", "Divide", "double", "long", "/"},
+ {"ScalarDivideColumn", "Divide", "double", "double", "/"},
+ {"ColumnDivideColumn", "Divide", "long", "double", "/"},
+ {"ColumnDivideColumn", "Divide", "double", "long", "/"},
+ {"ColumnDivideColumn", "Divide", "double", "double", "/"},
+
{"ColumnCompareScalar", "Equal", "long", "double", "=="},
{"ColumnCompareScalar", "Equal", "double", "double", "=="},
{"ColumnCompareScalar", "NotEqual", "long", "double", "!="},
@@ -518,7 +519,7 @@ public class GenVectorCode extends Task
private void generate() throws Exception {
System.out.println("Generating vector expression code");
for (String [] tdesc : templateExpansions) {
- if (tdesc[0].equals("ColumnArithmeticScalar")) {
+ if (tdesc[0].equals("ColumnArithmeticScalar") || tdesc[0].equals("ColumnDivideScalar")) {
generateColumnArithmeticScalar(tdesc);
} else if (tdesc[0].equals("ColumnCompareScalar")) {
generateColumnCompareScalar(tdesc);
@@ -530,13 +531,13 @@ public class GenVectorCode extends Task
generateFilterScalarCompareColumn(tdesc);
} else if (tdesc[0].equals("FilterColumnBetween")) {
generateFilterColumnBetween(tdesc);
- } else if (tdesc[0].equals("ScalarArithmeticColumn")) {
+ } else if (tdesc[0].equals("ScalarArithmeticColumn") || tdesc[0].equals("ScalarDivideColumn")) {
generateScalarArithmeticColumn(tdesc);
} else if (tdesc[0].equals("FilterColumnCompareColumn")) {
generateFilterColumnCompareColumn(tdesc);
} else if (tdesc[0].equals("ColumnCompareColumn")) {
generateColumnCompareColumn(tdesc);
- } else if (tdesc[0].equals("ColumnArithmeticColumn")) {
+ } else if (tdesc[0].equals("ColumnArithmeticColumn") || tdesc[0].equals("ColumnDivideColumn")) {
generateColumnArithmeticColumn(tdesc);
} else if (tdesc[0].equals("ColumnUnaryMinus")) {
generateColumnUnaryMinus(tdesc);
Modified: hive/branches/tez/beeline/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/beeline/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/beeline/pom.xml (original)
+++ hive/branches/tez/beeline/pom.xml Thu Dec 5 01:32:27 2013
@@ -113,9 +113,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLine.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original)
+++ hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLine.java Thu Dec 5 01:32:27 2013
@@ -224,6 +224,8 @@ public class BeeLine {
null),
new ReflectiveCommandHandler(this, new String[] {"call"},
null),
+ new ReflectiveCommandHandler(this, new String[] {"nullemptystring"},
+ new Completor[] {new BooleanCompletor()}),
};
Modified: hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java (original)
+++ hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java Thu Dec 5 01:32:27 2013
@@ -52,6 +52,7 @@ class BeeLineOpts implements Completor {
public static final String PROPERTY_PREFIX = "beeline.";
public static final String PROPERTY_NAME_EXIT =
PROPERTY_PREFIX + "system.exit";
+ public static final String DEFAULT_NULL_STRING = "NULL";
private final BeeLine beeLine;
private boolean autosave = false;
@@ -78,6 +79,9 @@ class BeeLineOpts implements Completor {
private boolean trimScripts = true;
private boolean allowMultiLineCommand = true;
+ //This can be set for old behavior of nulls printed as empty strings
+ private boolean nullEmptyString = false;
+
private final File rcFile = new File(saveDir(), "beeline.properties");
private String historyFile = new File(saveDir(), "history").getAbsolutePath();
@@ -443,5 +447,22 @@ class BeeLineOpts implements Completor {
this.allowMultiLineCommand = allowMultiLineCommand;
}
+ /**
+ * Use getNullString() to get the null string to be used.
+ * @return true if null representation should be an empty string
+ */
+ public boolean getNullEmptyString() {
+ return nullEmptyString;
+ }
+
+ public void setNullEmptyString(boolean nullStringEmpty) {
+ this.nullEmptyString = nullStringEmpty;
+ }
+
+ public String getNullString(){
+ return nullEmptyString ? "" : DEFAULT_NULL_STRING;
+ }
+
+
}
Modified: hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BufferedRows.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BufferedRows.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BufferedRows.java (original)
+++ hive/branches/tez/beeline/src/java/org/apache/hive/beeline/BufferedRows.java Thu Dec 5 01:32:27 2013
@@ -54,6 +54,11 @@ class BufferedRows extends Rows {
}
@Override
+ public String toString(){
+ return list.toString();
+ }
+
+ @Override
void normalizeWidths() {
int[] max = null;
for (Row row : list) {
Modified: hive/branches/tez/beeline/src/java/org/apache/hive/beeline/Rows.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/beeline/src/java/org/apache/hive/beeline/Rows.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/beeline/src/java/org/apache/hive/beeline/Rows.java (original)
+++ hive/branches/tez/beeline/src/java/org/apache/hive/beeline/Rows.java Thu Dec 5 01:32:27 2013
@@ -27,19 +27,23 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.text.NumberFormat;
+import java.util.Arrays;
import java.util.Iterator;
/**
* Abstract base class representing a set of rows to be displayed.
+ * Holds column values as strings
*/
abstract class Rows implements Iterator {
private final BeeLine beeLine;
final ResultSetMetaData rsMeta;
final Boolean[] primaryKeys;
final NumberFormat numberFormat;
+ private final String nullStr;
Rows(BeeLine beeLine, ResultSet rs) throws SQLException {
this.beeLine = beeLine;
+ nullStr = beeLine.getOpts().getNullString();
rsMeta = rs.getMetaData();
int count = rsMeta.getColumnCount();
primaryKeys = new Boolean[count];
@@ -125,6 +129,10 @@ abstract class Rows implements Iterator
inserted = false;
}
+ @Override
+ public String toString(){
+ return Arrays.asList(values).toString();
+ }
Row(int size, ResultSet rs) throws SQLException {
isMeta = false;
@@ -157,7 +165,8 @@ abstract class Rows implements Iterator
} else {
values[i] = rs.getString(i + 1);
}
- sizes[i] = values[i] == null ? 1 : values[i].length();
+ values[i] = values[i] == null ? nullStr : values[i];
+ sizes[i] = values[i].length();
}
}
}
Modified: hive/branches/tez/beeline/src/main/resources/BeeLine.properties
URL: http://svn.apache.org/viewvc/hive/branches/tez/beeline/src/main/resources/BeeLine.properties?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/beeline/src/main/resources/BeeLine.properties (original)
+++ hive/branches/tez/beeline/src/main/resources/BeeLine.properties Thu Dec 5 01:32:27 2013
@@ -67,6 +67,7 @@ help-tables: List all the tables in the
help-columns: List all the columns for the specified table
help-properties: Connect to the database specified in the properties file(s)
help-outputformat: Set the output format for displaying results (table,vertical,csv,tsv,xmlattrs,xmlelements)
+help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false.
jline-missing: The JLine jar was not found. Please ensure it is installed.
@@ -163,6 +164,7 @@ cmd-usage: Usage: java org.apache.hive.c
\ --autosave=[true/false] automatically save preferences\n \
\ --outputformat=[table/vertical/csv/tsv] format mode for result display\n \
\ --isolation=LEVEL set the transaction isolation level\n \
+\ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \
\ --help display this message
Modified: hive/branches/tez/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java (original)
+++ hive/branches/tez/beeline/src/test/org/apache/hive/beeline/src/test/TestBeeLineWithArgs.java Thu Dec 5 01:32:27 2013
@@ -23,9 +23,14 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.beeline.BeeLine;
import org.apache.hive.service.server.HiveServer2;
@@ -34,6 +39,8 @@ import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* TestBeeLineWithArgs - executes tests of the command-line arguments to BeeLine
*
@@ -42,6 +49,9 @@ import org.junit.Test;
public class TestBeeLineWithArgs {
// Default location of HiveServer2
final private static String JDBC_URL = BeeLine.BEELINE_DEFAULT_JDBC_URL + "localhost:10000";
+ private static final String tableName = "TestBeelineTable1";
+ private static final String tableComment = "Test table comment";
+
private static HiveServer2 hiveServer2;
@@ -67,6 +77,45 @@ public class TestBeeLineWithArgs {
System.err.println("Starting HiveServer2...");
hiveServer2.start();
Thread.sleep(1000);
+ createTable();
+
+ }
+
+ /**
+ * Create table for use by tests
+ * @throws ClassNotFoundException
+ * @throws SQLException
+ */
+ private static void createTable() throws ClassNotFoundException, SQLException {
+ Class.forName(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER);
+ Connection con = DriverManager.getConnection(JDBC_URL,"", "");
+
+ assertNotNull("Connection is null", con);
+ assertFalse("Connection should not be closed", con.isClosed());
+ Statement stmt = con.createStatement();
+ assertNotNull("Statement is null", stmt);
+
+ stmt.execute("set hive.support.concurrency = false");
+
+ HiveConf conf = new HiveConf();
+ String dataFileDir = conf.get("test.data.files").replace('\\', '/')
+ .replace("c:", "");
+ Path dataFilePath = new Path(dataFileDir, "kv1.txt");
+ // drop table. ignore error.
+ try {
+ stmt.execute("drop table " + tableName);
+ } catch (Exception ex) {
+ fail(ex.toString());
+ }
+
+ // create table
+ stmt.execute("create table " + tableName
+ + " (under_col int comment 'the under column', value string) comment '"
+ + tableComment + "'");
+
+ // load data
+ stmt.execute("load data local inpath '"
+ + dataFilePath.toString() + "' into table " + tableName);
}
/**
@@ -129,39 +178,12 @@ public class TestBeeLineWithArgs {
argList.add("-f");
argList.add(scriptFile.getAbsolutePath());
- if(shouldMatch) {
- try {
- String output = testCommandLineScript(argList);
- long elapsedTime = (System.currentTimeMillis() - startTime)/1000;
- String time = "(" + elapsedTime + "s)";
- if (output.contains(expectedPattern)) {
- System.out.println(">>> PASSED " + testName + " " + time);
- } else {
- System.err.println("Output: " + output);
- System.err.println(">>> FAILED " + testName + " (ERROR) " + time);
- Assert.fail(testName);
- }
- } catch (Throwable e) {
- e.printStackTrace();
- throw e;
- }
- } else {
- try {
- String output = testCommandLineScript(argList);
- long elapsedTime = (System.currentTimeMillis() - startTime)/1000;
- String time = "(" + elapsedTime + "s)";
- if (output.contains(expectedPattern)) {
- System.err.println("Output: " + output);
- System.err.println(">>> FAILED " + testName + " (ERROR) " + time);
- Assert.fail(testName);
- } else {
- System.out.println(">>> PASSED " + testName + " " + time);
- }
- } catch (Throwable e) {
- System.err.println("Exception: " + e.toString());
- e.printStackTrace();
- throw e;
- }
+ String output = testCommandLineScript(argList);
+ boolean matches = output.contains(expectedPattern);
+ if (shouldMatch != matches) {
+ //failed
+ fail(testName + ": Output" + output + " should" + (shouldMatch ? "" : " not") +
+ " contain " + expectedPattern);
}
scriptFile.delete();
}
@@ -212,6 +234,72 @@ public class TestBeeLineWithArgs {
testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, false, argList);
}
+
+ /**
+ * Select null from table , check how null is printed
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullDefault() throws Throwable {
+ final String TEST_NAME = "testNullDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "select null from " + tableName + " limit 1 ;\n";
+ final String EXPECTED_PATTERN = "NULL";
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL));
+ }
+
+ /**
+ * Select null from table , check if default null is printed differently
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullNonEmpty() throws Throwable {
+ final String TEST_NAME = "testNullNonDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n";
+ final String EXPECTED_PATTERN = "NULL";
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(JDBC_URL));
+ }
+
+ /**
+ * Select null from table , check if setting null to empty string works.
+ * Original beeline/sqlline used to print nulls as empty strings
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullEmpty() throws Throwable {
+ final String TEST_NAME = "testNullNonDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
+ final String EXPECTED_PATTERN = "'abc','','def'";
+
+ List<String> argList = getBaseArgs(JDBC_URL);
+ argList.add("--outputformat=csv");
+
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ }
+
+ /**
+ * Select null from table , check if setting null to empty string works - Using beeling cmd line
+ * argument.
+ * Original beeline/sqlline used to print nulls as empty strings
+ * Print PASSED or FAILED
+ */
+ @Test
+ public void testNullEmptyCmdArg() throws Throwable {
+ final String TEST_NAME = "testNullNonDefault";
+ final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
+ "select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
+ //final String EXPECTED_PATTERN = "| abc | | def |";
+ final String EXPECTED_PATTERN = "'abc','','def'";
+
+ List<String> argList = getBaseArgs(JDBC_URL);
+ argList.add("--nullemptystring=true");
+ argList.add("--outputformat=csv");
+
+ testScriptFile(TEST_NAME, SCRIPT_TEXT, EXPECTED_PATTERN, true, argList);
+ }
+
/**
* Attempt to execute a missing script file with the -f option to BeeLine
* Print PASSED or FAILED
@@ -239,7 +327,7 @@ public class TestBeeLineWithArgs {
if (output.contains(EXPECTED_PATTERN)) {
System.err.println("Output: " + output);
System.err.println(">>> FAILED " + TEST_NAME + " (ERROR) " + time);
- Assert.fail(TEST_NAME);
+ fail(TEST_NAME);
} else {
System.out.println(">>> PASSED " + TEST_NAME + " " + time);
}
Modified: hive/branches/tez/cli/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/cli/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/cli/pom.xml (original)
+++ hive/branches/tez/cli/pom.xml Thu Dec 5 01:32:27 2013
@@ -114,9 +114,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/common/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/common/pom.xml (original)
+++ hive/branches/tez/common/pom.xml Thu Dec 5 01:32:27 2013
@@ -77,9 +77,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Dec 5 01:32:27 2013
@@ -645,6 +645,9 @@ public class HiveConf extends Configurat
HIVE_STATS_MAP_NUM_ENTRIES("hive.stats.map.num.entries", 10),
// to accurately compute statistics for GROUPBY map side parallelism needs to be known
HIVE_STATS_MAP_SIDE_PARALLELISM("hive.stats.map.parallelism", 1),
+ // statistics annotation fetches column statistics for all required columns and for all
+ // required partitions which can be very expensive sometimes
+ HIVE_STATS_FETCH_COLUMN_STATS("hive.stats.fetch.column.stats", false),
// Concurrency
HIVE_SUPPORT_CONCURRENCY("hive.support.concurrency", false),
Modified: hive/branches/tez/contrib/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/contrib/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/contrib/pom.xml (original)
+++ hive/branches/tez/contrib/pom.xml Thu Dec 5 01:32:27 2013
@@ -72,9 +72,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/hbase-handler/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hbase-handler/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/hbase-handler/pom.xml (original)
+++ hive/branches/tez/hbase-handler/pom.xml Thu Dec 5 01:32:27 2013
@@ -87,9 +87,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/hcatalog/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/pom.xml (original)
+++ hive/branches/tez/hcatalog/pom.xml Thu Dec 5 01:32:27 2013
@@ -49,9 +49,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/pom.xml (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/pom.xml Thu Dec 5 01:32:27 2013
@@ -29,7 +29,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>hive-hbase-storage-handler</artifactId>
+ <artifactId>hive-hcatalog-hbase-storage-handler</artifactId>
<packaging>jar</packaging>
<name>Hive HCatalog HBase Storage Handler</name>
Modified: hive/branches/tez/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateTableDesc.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateTableDesc.java (original)
+++ hive/branches/tez/hcatalog/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatCreateTableDesc.java Thu Dec 5 01:32:27 2013
@@ -31,7 +31,11 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -39,7 +43,6 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hcatalog.common.HCatException;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -496,12 +499,16 @@ public class HCatCreateTableDesc {
desc.fileFormat = fileFormat;
if ("SequenceFile".equalsIgnoreCase(fileFormat)) {
desc.inputformat = SequenceFileInputFormat.class.getName();
- desc.outputformat = SequenceFileOutputFormat.class
+ desc.outputformat = HiveSequenceFileOutputFormat.class
.getName();
} else if ("RCFile".equalsIgnoreCase(fileFormat)) {
desc.inputformat = RCFileInputFormat.class.getName();
desc.outputformat = RCFileOutputFormat.class.getName();
desc.serde = ColumnarSerDe.class.getName();
+ } else if ("orcfile".equalsIgnoreCase(fileFormat)) {
+ desc.inputformat = OrcInputFormat.class.getName();
+ desc.outputformat = OrcOutputFormat.class.getName();
+ desc.serde = OrcSerde.class.getName();
}
desc.storageHandler = StringUtils.EMPTY;
} else if (!StringUtils.isEmpty(storageHandler)) {
@@ -511,7 +518,7 @@ public class HCatCreateTableDesc {
LOG.info("Using text file format for the table.");
desc.inputformat = TextInputFormat.class.getName();
LOG.info("Table input format:" + desc.inputformat);
- desc.outputformat = IgnoreKeyTextOutputFormat.class
+ desc.outputformat = HiveIgnoreKeyTextOutputFormat.class
.getName();
LOG.info("Table output format:" + desc.outputformat);
}
Modified: hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (original)
+++ hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java Thu Dec 5 01:32:27 2013
@@ -30,7 +30,10 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.api.PartitionEventType;
-import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
@@ -109,6 +112,7 @@ public class TestHCatClient {
String db = "testdb";
String tableOne = "testTable1";
String tableTwo = "testTable2";
+ String tableThree = "testTable3";
HCatClient client = HCatClient.create(new Configuration(hcatConf));
client.dropDatabase(db, true, HCatClient.DropDBMode.CASCADE);
@@ -159,8 +163,21 @@ public class TestHCatClient {
assertTrue(table2.getInputFileFormat().equalsIgnoreCase(
TextInputFormat.class.getName()));
assertTrue(table2.getOutputFileFormat().equalsIgnoreCase(
- IgnoreKeyTextOutputFormat.class.getName()));
+ HiveIgnoreKeyTextOutputFormat.class.getName()));
assertEquals((expectedDir + "/" + db + ".db/" + tableTwo).toLowerCase(), table2.getLocation().toLowerCase());
+
+ HCatCreateTableDesc tableDesc3 = HCatCreateTableDesc.create(db,
+ tableThree, cols).fileFormat("orcfile").build();
+ client.createTable(tableDesc3);
+ HCatTable table3 = client.getTable(db, tableThree);
+ assertTrue(table3.getInputFileFormat().equalsIgnoreCase(
+ OrcInputFormat.class.getName()));
+ assertTrue(table3.getOutputFileFormat().equalsIgnoreCase(
+ OrcOutputFormat.class.getName()));
+ assertTrue(table3.getSerdeLib().equalsIgnoreCase(
+ OrcSerde.class.getName()));
+ assertTrue(table1.getCols().equals(cols));
+
client.close();
}
Modified: hive/branches/tez/hwi/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/hwi/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/hwi/pom.xml (original)
+++ hive/branches/tez/hwi/pom.xml Thu Dec 5 01:32:27 2013
@@ -96,9 +96,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<!-- dependencies are always listed in sorted order by groupId, artifectId -->
<dependency>
Modified: hive/branches/tez/jdbc/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/jdbc/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/jdbc/pom.xml (original)
+++ hive/branches/tez/jdbc/pom.xml Thu Dec 5 01:32:27 2013
@@ -85,9 +85,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/metastore/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/metastore/pom.xml (original)
+++ hive/branches/tez/metastore/pom.xml Thu Dec 5 01:32:27 2013
@@ -130,9 +130,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql
URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql (original)
+++ hive/branches/tez/metastore/scripts/upgrade/mysql/hive-schema-0.13.0.mysql.sql Thu Dec 5 01:32:27 2013
@@ -757,7 +757,7 @@ CREATE TABLE IF NOT EXISTS `DELEGATION_T
CREATE TABLE IF NOT EXISTS `VERSION` (
`VER_ID` BIGINT NOT NULL,
`SCHEMA_VERSION` VARCHAR(127) NOT NULL,
- `COMMENT` VARCHAR(255),
+ `VERSION_COMMENT` VARCHAR(255),
PRIMARY KEY (`VER_ID`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
Modified: hive/branches/tez/packaging/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/packaging/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/packaging/pom.xml (original)
+++ hive/branches/tez/packaging/pom.xml Thu Dec 5 01:32:27 2013
@@ -133,7 +133,7 @@
</dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
- <artifactId>hive-hbase-storage-handler</artifactId>
+ <artifactId>hive-hcatalog-hbase-storage-handler</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
Modified: hive/branches/tez/packaging/src/main/assembly/bin.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/packaging/src/main/assembly/bin.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/packaging/src/main/assembly/bin.xml (original)
+++ hive/branches/tez/packaging/src/main/assembly/bin.xml Thu Dec 5 01:32:27 2013
@@ -61,7 +61,7 @@
<useProjectArtifact>false</useProjectArtifact>
<useStrictFiltering>true</useStrictFiltering>
<includes>
- <include>org.apache.hive.hcatalog:hive-hbase-storage-handler</include>
+ <include>org.apache.hive.hcatalog:hive-hcatalog-hbase-storage-handler</include>
</includes>
</dependencySet>
<!-- TODO 0.12 release the webhcat server lib directory had all transitive jars
Modified: hive/branches/tez/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/pom.xml (original)
+++ hive/branches/tez/pom.xml Thu Dec 5 01:32:27 2013
@@ -373,11 +373,6 @@
<version>${httpcomponents.core.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.mina</groupId>
- <artifactId>mina-core</artifactId>
- <version>${mina.version}</version>
- </dependency>
- <dependency>
<groupId>org.apache.velocity</groupId>
<artifactId>velocity</artifactId>
<version>${velocity.version}</version>
@@ -692,7 +687,7 @@
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<reuseForks>false</reuseForks>
<failIfNoTests>false</failIfNoTests>
- <argLine></argLine>
+ <argLine>-Xmx1024m -XX:MaxPermSize=256m</argLine>
<additionalClasspathElements>
<additionalClasspathElement>${test.tmp.dir}/conf</additionalClasspathElement>
<additionalClasspathElement>${basedir}/${hive.path.to.root}/conf</additionalClasspathElement>
Modified: hive/branches/tez/ql/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/pom.xml (original)
+++ hive/branches/tez/ql/pom.xml Thu Dec 5 01:32:27 2013
@@ -210,12 +210,6 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.mina</groupId>
- <artifactId>mina-core</artifactId>
- <version>${mina.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito-all.version}</version>
@@ -226,9 +220,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -433,6 +424,12 @@
<include>com.google.guava:guava</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.esotericsoftware</pattern>
+ <shadedPattern>org.apache.hive.com.esotericsoftware</shadedPattern>
+ </relocation>
+ </relocations>
</configuration>
</execution>
</executions>
Modified: hive/branches/tez/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java (original)
+++ hive/branches/tez/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java Thu Dec 5 01:32:27 2013
@@ -9913,6 +9913,10 @@ public final class OrcProto {
* <code>VARCHAR = 16;</code>
*/
VARCHAR(16, 16),
+ /**
+ * <code>CHAR = 17;</code>
+ */
+ CHAR(17, 17),
;
/**
@@ -9983,6 +9987,10 @@ public final class OrcProto {
* <code>VARCHAR = 16;</code>
*/
public static final int VARCHAR_VALUE = 16;
+ /**
+ * <code>CHAR = 17;</code>
+ */
+ public static final int CHAR_VALUE = 17;
public final int getNumber() { return value; }
@@ -10006,6 +10014,7 @@ public final class OrcProto {
case 14: return DECIMAL;
case 15: return DATE;
case 16: return VARCHAR;
+ case 17: return CHAR;
default: return null;
}
}
@@ -16767,40 +16776,40 @@ public final class OrcProto {
"9\n\007streams\030\001 \003(\0132(.org.apache.hadoop.hiv" +
"e.ql.io.orc.Stream\022A\n\007columns\030\002 \003(\01320.or" +
"g.apache.hadoop.hive.ql.io.orc.ColumnEnc" +
- "oding\"\356\002\n\004Type\0229\n\004kind\030\001 \002(\0162+.org.apach" +
+ "oding\"\370\002\n\004Type\0229\n\004kind\030\001 \002(\0162+.org.apach" +
"e.hadoop.hive.ql.io.orc.Type.Kind\022\024\n\010sub" +
"types\030\002 \003(\rB\002\020\001\022\022\n\nfieldNames\030\003 \003(\t\022\025\n\rm" +
"aximumLength\030\004 \001(\r\022\021\n\tprecision\030\005 \001(\r\022\r\n" +
- "\005scale\030\006 \001(\r\"\307\001\n\004Kind\022\013\n\007BOOLEAN\020\000\022\010\n\004BY" +
+ "\005scale\030\006 \001(\r\"\321\001\n\004Kind\022\013\n\007BOOLEAN\020\000\022\010\n\004BY" +
"TE\020\001\022\t\n\005SHORT\020\002\022\007\n\003INT\020\003\022\010\n\004LONG\020\004\022\t\n\005FL",
"OAT\020\005\022\n\n\006DOUBLE\020\006\022\n\n\006STRING\020\007\022\n\n\006BINARY\020" +
"\010\022\r\n\tTIMESTAMP\020\t\022\010\n\004LIST\020\n\022\007\n\003MAP\020\013\022\n\n\006S" +
"TRUCT\020\014\022\t\n\005UNION\020\r\022\013\n\007DECIMAL\020\016\022\010\n\004DATE\020" +
- "\017\022\013\n\007VARCHAR\020\020\"x\n\021StripeInformation\022\016\n\006o" +
- "ffset\030\001 \001(\004\022\023\n\013indexLength\030\002 \001(\004\022\022\n\ndata" +
- "Length\030\003 \001(\004\022\024\n\014footerLength\030\004 \001(\004\022\024\n\014nu" +
- "mberOfRows\030\005 \001(\004\"/\n\020UserMetadataItem\022\014\n\004" +
- "name\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"X\n\020StripeStati" +
- "stics\022D\n\010colStats\030\001 \003(\01322.org.apache.had" +
- "oop.hive.ql.io.orc.ColumnStatistics\"S\n\010M",
- "etadata\022G\n\013stripeStats\030\001 \003(\01322.org.apach" +
- "e.hadoop.hive.ql.io.orc.StripeStatistics" +
- "\"\356\002\n\006Footer\022\024\n\014headerLength\030\001 \001(\004\022\025\n\rcon" +
- "tentLength\030\002 \001(\004\022D\n\007stripes\030\003 \003(\01323.org." +
- "apache.hadoop.hive.ql.io.orc.StripeInfor" +
- "mation\0225\n\005types\030\004 \003(\0132&.org.apache.hadoo" +
- "p.hive.ql.io.orc.Type\022D\n\010metadata\030\005 \003(\0132" +
- "2.org.apache.hadoop.hive.ql.io.orc.UserM" +
- "etadataItem\022\024\n\014numberOfRows\030\006 \001(\004\022F\n\nsta" +
- "tistics\030\007 \003(\01322.org.apache.hadoop.hive.q",
- "l.io.orc.ColumnStatistics\022\026\n\016rowIndexStr" +
- "ide\030\010 \001(\r\"\305\001\n\nPostScript\022\024\n\014footerLength" +
- "\030\001 \001(\004\022F\n\013compression\030\002 \001(\01621.org.apache" +
- ".hadoop.hive.ql.io.orc.CompressionKind\022\034" +
- "\n\024compressionBlockSize\030\003 \001(\004\022\023\n\007version\030" +
- "\004 \003(\rB\002\020\001\022\026\n\016metadataLength\030\005 \001(\004\022\016\n\005mag" +
- "ic\030\300> \001(\t*:\n\017CompressionKind\022\010\n\004NONE\020\000\022\010" +
- "\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003"
+ "\017\022\013\n\007VARCHAR\020\020\022\010\n\004CHAR\020\021\"x\n\021StripeInform" +
+ "ation\022\016\n\006offset\030\001 \001(\004\022\023\n\013indexLength\030\002 \001" +
+ "(\004\022\022\n\ndataLength\030\003 \001(\004\022\024\n\014footerLength\030\004" +
+ " \001(\004\022\024\n\014numberOfRows\030\005 \001(\004\"/\n\020UserMetada" +
+ "taItem\022\014\n\004name\030\001 \002(\t\022\r\n\005value\030\002 \002(\014\"X\n\020S" +
+ "tripeStatistics\022D\n\010colStats\030\001 \003(\01322.org." +
+ "apache.hadoop.hive.ql.io.orc.ColumnStati",
+ "stics\"S\n\010Metadata\022G\n\013stripeStats\030\001 \003(\01322" +
+ ".org.apache.hadoop.hive.ql.io.orc.Stripe" +
+ "Statistics\"\356\002\n\006Footer\022\024\n\014headerLength\030\001 " +
+ "\001(\004\022\025\n\rcontentLength\030\002 \001(\004\022D\n\007stripes\030\003 " +
+ "\003(\01323.org.apache.hadoop.hive.ql.io.orc.S" +
+ "tripeInformation\0225\n\005types\030\004 \003(\0132&.org.ap" +
+ "ache.hadoop.hive.ql.io.orc.Type\022D\n\010metad" +
+ "ata\030\005 \003(\01322.org.apache.hadoop.hive.ql.io" +
+ ".orc.UserMetadataItem\022\024\n\014numberOfRows\030\006 " +
+ "\001(\004\022F\n\nstatistics\030\007 \003(\01322.org.apache.had",
+ "oop.hive.ql.io.orc.ColumnStatistics\022\026\n\016r" +
+ "owIndexStride\030\010 \001(\r\"\305\001\n\nPostScript\022\024\n\014fo" +
+ "oterLength\030\001 \001(\004\022F\n\013compression\030\002 \001(\01621." +
+ "org.apache.hadoop.hive.ql.io.orc.Compres" +
+ "sionKind\022\034\n\024compressionBlockSize\030\003 \001(\004\022\023" +
+ "\n\007version\030\004 \003(\rB\002\020\001\022\026\n\016metadataLength\030\005 " +
+ "\001(\004\022\016\n\005magic\030\300> \001(\t*:\n\017CompressionKind\022\010" +
+ "\n\004NONE\020\000\022\010\n\004ZLIB\020\001\022\n\n\006SNAPPY\020\002\022\007\n\003LZO\020\003"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Thu Dec 5 01:32:27 2013
@@ -901,8 +901,19 @@ public class Driver implements CommandPr
perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.RELEASE_LOCKS);
}
- public CommandProcessorResponse run(String command) throws CommandNeedRetryException {
- CommandProcessorResponse cpr = runInternal(command);
+ public CommandProcessorResponse run(String command)
+ throws CommandNeedRetryException {
+ return run(command, false);
+ }
+
+ public CommandProcessorResponse run()
+ throws CommandNeedRetryException {
+ return run(null, true);
+ }
+
+ public CommandProcessorResponse run(String command, boolean alreadyCompiled)
+ throws CommandNeedRetryException {
+ CommandProcessorResponse cpr = runInternal(command, alreadyCompiled);
if(cpr.getResponseCode() == 0) {
return cpr;
}
@@ -957,7 +968,25 @@ public class Driver implements CommandPr
}
return cpr;
}
- private CommandProcessorResponse runInternal(String command) throws CommandNeedRetryException {
+
+ public CommandProcessorResponse compileAndRespond(String command) {
+ return new CommandProcessorResponse(compileInternal(command),
+ errorMessage, SQLState);
+ }
+
+ private int compileInternal(String command) {
+ int ret;
+ synchronized (compileMonitor) {
+ ret = compile(command);
+ }
+ if (ret != 0) {
+ releaseLocks(ctx.getHiveLocks());
+ }
+ return ret;
+ }
+
+ private CommandProcessorResponse runInternal(String command, boolean alreadyCompiled)
+ throws CommandNeedRetryException {
errorMessage = null;
SQLState = null;
downstreamError = null;
@@ -990,12 +1019,11 @@ public class Driver implements CommandPr
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TIME_TO_SUBMIT);
int ret;
- synchronized (compileMonitor) {
- ret = compile(command);
- }
- if (ret != 0) {
- releaseLocks(ctx.getHiveLocks());
- return new CommandProcessorResponse(ret, errorMessage, SQLState);
+ if (!alreadyCompiled) {
+ ret = compileInternal(command);
+ if (ret != 0) {
+ return new CommandProcessorResponse(ret, errorMessage, SQLState);
+ }
}
boolean requireLock = false;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java Thu Dec 5 01:32:27 2013
@@ -550,7 +550,7 @@ public class VectorizationContext {
if (udf instanceof GenericUDFBetween) {
return getBetweenFilterExpression(childExpr, mode);
} else if (udf instanceof GenericUDFIn) {
- return getInFilterExpression(childExpr);
+ return getInExpression(childExpr, mode);
} else if (udf instanceof GenericUDFOPPositive) {
return getIdentityExpression(childExpr);
} else if (udf instanceof GenericUDFBridge) {
@@ -575,11 +575,9 @@ public class VectorizationContext {
}
/**
- * Create a filter expression for column IN ( <list-of-constants> )
- * @param childExpr
- * @return
+ * Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
*/
- private VectorExpression getInFilterExpression(List<ExprNodeDesc> childExpr)
+ private VectorExpression getInExpression(List<ExprNodeDesc> childExpr, Mode mode)
throws HiveException {
ExprNodeDesc colExpr = childExpr.get(0);
String colType = colExpr.getTypeString();
@@ -601,48 +599,41 @@ public class VectorizationContext {
// determine class
Class<?> cl = null;
if (isIntFamily(colType)) {
- cl = FilterLongColumnInList.class;
+ cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
long[] inVals = new long[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getIntFamilyScalarAsLong((ExprNodeConstantDesc) childrenForInList.get(i));
}
- FilterLongColumnInList f = (FilterLongColumnInList)
- createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
- f.setInListValues(inVals);
- expr = f;
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
+ ((ILongInExpr) expr).setInListValues(inVals);
} else if (colType.equals("timestamp")) {
- cl = FilterLongColumnInList.class;
+ cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
long[] inVals = new long[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getTimestampScalar(childrenForInList.get(i));
}
- FilterLongColumnInList f = (FilterLongColumnInList)
- createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
- f.setInListValues(inVals);
- expr = f;
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
+ ((ILongInExpr) expr).setInListValues(inVals);
} else if (colType.equals("string")) {
- cl = FilterStringColumnInList.class;
+ cl = (mode == Mode.FILTER ? FilterStringColumnInList.class : StringColumnInList.class);
byte[][] inVals = new byte[childrenForInList.size()][];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getStringScalarAsByteArray((ExprNodeConstantDesc) childrenForInList.get(i));
}
- FilterStringColumnInList f =(FilterStringColumnInList)
- createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
- f.setInListValues(inVals);
- expr = f;
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
+ ((IStringInExpr) expr).setInListValues(inVals);
} else if (isFloatFamily(colType)) {
- cl = FilterDoubleColumnInList.class;
+ cl = (mode == Mode.FILTER ? FilterDoubleColumnInList.class : DoubleColumnInList.class);
double[] inValsD = new double[childrenForInList.size()];
for (int i = 0; i != inValsD.length; i++) {
inValsD[i] = getNumericScalarAsDouble(childrenForInList.get(i));
}
- FilterDoubleColumnInList f = (FilterDoubleColumnInList)
- createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
- f.setInListValues(inValsD);
- expr = f;
- } else {
- throw new HiveException("Type " + colType + " not supported for IN in vectorized mode");
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION);
+ ((IDoubleInExpr) expr).setInListValues(inValsD);
}
+
+ // Return the desired VectorExpression if found. Otherwise, return null to cause
+ // execution to fall back to row mode.
return expr;
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDoubleColumnInList.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDoubleColumnInList.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDoubleColumnInList.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDoubleColumnInList.java Thu Dec 5 01:32:27 2013
@@ -34,7 +34,7 @@ import java.util.regex.Pattern;
/**
* Evaluate IN filter on a batch for a vector of doubles.
*/
-public class FilterDoubleColumnInList extends VectorExpression {
+public class FilterDoubleColumnInList extends VectorExpression implements IDoubleInExpr {
private static final long serialVersionUID = 1L;
private int inputCol;
private double[] inListValues;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterLongColumnInList.java Thu Dec 5 01:32:27 2013
@@ -33,7 +33,8 @@ import java.util.regex.Pattern;
/**
* Evaluate IN filter on a batch for a vector of longs.
*/
-public class FilterLongColumnInList extends VectorExpression {
+public class FilterLongColumnInList extends VectorExpression implements ILongInExpr {
+
private static final long serialVersionUID = 1L;
private int inputCol;
private long[] inListValues;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColumnInList.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColumnInList.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColumnInList.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStringColumnInList.java Thu Dec 5 01:32:27 2013
@@ -37,7 +37,7 @@ import java.util.regex.Pattern;
* the inner loop, and there is a hash table implemented
* with Cuckoo hashing that has fast lookup to do the IN test.
*/
-public class FilterStringColumnInList extends VectorExpression {
+public class FilterStringColumnInList extends VectorExpression implements IStringInExpr {
private static final long serialVersionUID = 1L;
private int inputCol;
private byte[][] inListValues;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongColumn.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongColumn.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongColumn.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongColumn.java Thu Dec 5 01:32:27 2013
@@ -77,21 +77,32 @@ public class LongColDivideLongColumn ext
* more inputs are null. This is to improve speed by avoiding
* conditional checks in the inner loop.
*/
+ boolean hasDivBy0 = false;
if (inputColVector1.isRepeating && inputColVector2.isRepeating) {
- outputVector[0] = vector1[0] / (double) vector2[0];
+ long denom = vector2[0];
+ outputVector[0] = vector1[0] / (double) denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
} else if (inputColVector1.isRepeating) {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[0] / (double) vector2[i];
+ long denom = vector2[i];
+ outputVector[i] = vector1[0] / (double) denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = vector1[0] / (double) vector2[i];
+ long denom = vector2[i];
+ outputVector[i] = vector1[0] / (double) denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
}
}
} else if (inputColVector2.isRepeating) {
- if (batch.selectedInUse) {
+ if (vector2[0] == 0) {
+ outputColVector.noNulls = false;
+ outputColVector.isRepeating = true;
+ outputColVector.isNull[0] = true;
+ } else if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
outputVector[i] = vector1[i] / (double) vector2[0];
@@ -105,11 +116,15 @@ public class LongColDivideLongColumn ext
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = vector1[i] / (double) vector2[i];
+ long denom = vector2[i];
+ outputVector[i] = vector1[i] / (double) denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = vector1[i] / (double) vector2[i];
+ long denom = vector2[i];
+ outputVector[i] = vector1[i] / (double) denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
}
}
}
@@ -120,7 +135,12 @@ public class LongColDivideLongColumn ext
* in complex arithmetic expressions like col2 / (col1 - 1)
* in the case when some col1 entries are null.
*/
- NullUtil.setNullDataEntriesDouble(outputColVector, batch.selectedInUse, sel, n);
+ if (!hasDivBy0) {
+ NullUtil.setNullDataEntriesDouble(outputColVector, batch.selectedInUse, sel, n);
+ } else {
+ NullUtil.setNullAndDivBy0DataEntriesDouble(
+ outputColVector, batch.selectedInUse, sel, n, inputColVector2);
+ }
}
@Override
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongScalar.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongScalar.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongScalar.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongColDivideLongScalar.java Thu Dec 5 01:32:27 2013
@@ -68,9 +68,13 @@ public class LongColDivideLongScalar ext
return;
}
- if (inputColVector.isRepeating) {
+ if (value == 0) {
+ // Denominator is zero, convert the batch to nulls
+ outputColVector.noNulls = false;
+ outputColVector.isRepeating = true;
+ outputIsNull[0] = true;
+ } else if (inputColVector.isRepeating) {
outputVector[0] = vector[0] / (double) value;
-
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
} else if (inputColVector.noNulls) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarDivideLongColumn.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarDivideLongColumn.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarDivideLongColumn.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/LongScalarDivideLongColumn.java Thu Dec 5 01:32:27 2013
@@ -68,8 +68,11 @@ public class LongScalarDivideLongColumn
return;
}
+ boolean hasDivBy0 = false;
if (inputColVector.isRepeating) {
- outputVector[0] = value / vector[0];
+ long denom = vector[0];
+ outputVector[0] = value / denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
// Even if there are no nulls, we always copy over entry 0. Simplifies code.
outputIsNull[0] = inputIsNull[0];
@@ -77,23 +80,31 @@ public class LongScalarDivideLongColumn
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = value / vector[i];
+ long denom = vector[i];
+ outputVector[i] = value / denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = value / vector[i];
+ long denom = vector[i];
+ outputVector[i] = value / denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
}
}
} else /* there are nulls */ {
if (batch.selectedInUse) {
for(int j = 0; j != n; j++) {
int i = sel[j];
- outputVector[i] = value / vector[i];
+ long denom = vector[i];
+ outputVector[i] = value / denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
outputIsNull[i] = inputIsNull[i];
}
} else {
for(int i = 0; i != n; i++) {
- outputVector[i] = value / vector[i];
+ long denom = vector[i];
+ outputVector[i] = value / denom;
+ hasDivBy0 = hasDivBy0 || (denom == 0);
}
System.arraycopy(inputIsNull, 0, outputIsNull, 0, n);
}
@@ -103,7 +114,12 @@ public class LongScalarDivideLongColumn
* Unlike other col-scalar operations, this one doesn't benefit from carrying
* over NaN values from the input array.
*/
- NullUtil.setNullDataEntriesDouble(outputColVector, batch.selectedInUse, sel, n);
+ if (!hasDivBy0) {
+ NullUtil.setNullDataEntriesDouble(outputColVector, batch.selectedInUse, sel, n);
+ } else {
+ NullUtil.setNullAndDivBy0DataEntriesDouble(
+ outputColVector, batch.selectedInUse, sel, n, inputColVector);
+ }
}
@Override
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/NullUtil.java Thu Dec 5 01:32:27 2013
@@ -27,7 +27,6 @@ import org.apache.hadoop.hive.ql.exec.ve
* Utility functions to handle null propagation.
*/
public class NullUtil {
-
/*
* Set the data value for all NULL entries to the designated NULL_VALUE.
*/
@@ -42,7 +41,7 @@ public class NullUtil {
int i = sel[j];
if(v.isNull[i]) {
v.vector[i] = LongColumnVector.NULL_VALUE;
- }
+ }
}
} else {
for (int i = 0; i != n; i++) {
@@ -50,7 +49,7 @@ public class NullUtil {
v.vector[i] = LongColumnVector.NULL_VALUE;
}
}
- }
+ }
}
// for use by Column-Scalar and Scalar-Column arithmetic for null propagation
@@ -77,28 +76,84 @@ public class NullUtil {
} else if (selectedInUse) {
for (int j = 0; j != n; j++) {
int i = sel[j];
- if(v.isNull[i]) {
+ if (v.isNull[i]) {
v.vector[i] = DoubleColumnVector.NULL_VALUE;
- }
+ }
}
} else {
for (int i = 0; i != n; i++) {
- if(v.isNull[i]) {
+ if (v.isNull[i]) {
v.vector[i] = DoubleColumnVector.NULL_VALUE;
}
}
- }
+ }
}
-
+
+ /*
+ * Set the data value for all NULL entries, as well as those coming from division by zero,
+ * to NaN. Assumes there are entries coming from division by zero.
+ * We assume that infinities do not appear legally in the result, so we can replace all of them.
+ */
+ public static void setNullAndDivBy0DataEntriesDouble(
+ DoubleColumnVector v, boolean selectedInUse, int[] sel, int n, LongColumnVector denoms) {
+ assert v.isRepeating || !denoms.isRepeating;
+ v.noNulls = false;
+ long[] vector = denoms.vector;
+ if (v.isRepeating && (v.isNull[0] = (v.isNull[0] || vector[0] == 0))) {
+ v.vector[0] = DoubleColumnVector.NULL_VALUE;
+ } else if (selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (v.isNull[i] = (v.isNull[i] || vector[i] == 0)) {
+ v.vector[i] = DoubleColumnVector.NULL_VALUE;
+ }
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ if (v.isNull[i] = (v.isNull[i] || vector[i] == 0)) {
+ v.vector[i] = DoubleColumnVector.NULL_VALUE;
+ }
+ }
+ }
+ }
+
+ /*
+ * Set the data value for all NULL entries, as well as those coming from division by zero,
+ * to NaN. Assumes there are entries coming from division by zero.
+ * We assume that infinities do not appear legally in the result, so we can replace all of them.
+ */
+ public static void setNullAndDivBy0DataEntriesDouble(
+ DoubleColumnVector v, boolean selectedInUse, int[] sel, int n, DoubleColumnVector denoms) {
+ assert v.isRepeating || !denoms.isRepeating;
+ v.noNulls = false;
+ double[] vector = denoms.vector;
+ if (v.isRepeating && (v.isNull[0] = (v.isNull[0] || vector[0] == 0))) {
+ v.vector[0] = DoubleColumnVector.NULL_VALUE;
+ } else if (selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (v.isNull[i] = (v.isNull[i] || vector[i] == 0)) {
+ v.vector[i] = DoubleColumnVector.NULL_VALUE;
+ }
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ if (v.isNull[i] = (v.isNull[i] || vector[i] == 0)) {
+ v.vector[i] = DoubleColumnVector.NULL_VALUE;
+ }
+ }
+ }
+ }
+
/*
* Propagate null values for a two-input operator.
*/
public static void propagateNullsColCol(ColumnVector inputColVector1,
- ColumnVector inputColVector2, ColumnVector outputColVector, int[] sel,
+ ColumnVector inputColVector2, ColumnVector outputColVector, int[] sel,
int n, boolean selectedInUse) {
outputColVector.noNulls = inputColVector1.noNulls && inputColVector2.noNulls;
-
+
if (inputColVector1.noNulls && !inputColVector2.noNulls) {
if (inputColVector2.isRepeating) {
outputColVector.isNull[0] = inputColVector2.isNull[0];
@@ -144,11 +199,11 @@ public class NullUtil {
outputColVector.isNull[i] = inputColVector2.isNull[i];
}
} else {
-
+
// copy nulls from the non-repeating side
System.arraycopy(inputColVector2.isNull, 0, outputColVector.isNull, 0, n);
}
- }
+ }
} else if (!inputColVector1.isRepeating && inputColVector2.isRepeating) {
if (inputColVector2.isNull[0]) {
outputColVector.isNull[0] = true;
@@ -161,11 +216,10 @@ public class NullUtil {
outputColVector.isNull[i] = inputColVector1.isNull[i];
}
} else {
-
// copy nulls from the non-repeating side
System.arraycopy(inputColVector1.isNull, 0, outputColVector.isNull, 0, n);
}
- }
+ }
} else { // neither side is repeating
if (selectedInUse) {
for(int j = 0; j != n; j++) {
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java Thu Dec 5 01:32:27 2013
@@ -789,6 +789,7 @@ class ColumnStatisticsImpl implements Co
case DOUBLE:
return new DoubleStatisticsImpl();
case STRING:
+ case CHAR:
case VARCHAR:
return new StringStatisticsImpl();
case DECIMAL:
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java Thu Dec 5 01:32:27 2013
@@ -483,6 +483,9 @@ final class OrcStruct implements Writabl
return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
case STRING:
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ case CHAR:
+ return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ (PrimitiveTypeInfo) info);
case VARCHAR:
return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
(PrimitiveTypeInfo) info);
@@ -533,6 +536,13 @@ final class OrcStruct implements Writabl
return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
case STRING:
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ case CHAR:
+ if (!type.hasMaximumLength()) {
+ throw new UnsupportedOperationException(
+ "Illegal use of char type without length in ORC type definition.");
+ }
+ return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ TypeInfoFactory.getCharTypeInfo(type.getMaximumLength()));
case VARCHAR:
if (!type.hasMaximumLength()) {
throw new UnsupportedOperationException(
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Thu Dec 5 01:32:27 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.io.sarg
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
@@ -1471,6 +1472,34 @@ class RecordReaderImpl implements Record
}
}
+ private static class CharTreeReader extends StringTreeReader {
+ int maxLength;
+
+ CharTreeReader(Path path, int columnId, int maxLength) {
+ super(path, columnId);
+ this.maxLength = maxLength;
+ }
+
+ @Override
+ Object next(Object previous) throws IOException {
+ HiveCharWritable result = null;
+ if (previous == null) {
+ result = new HiveCharWritable();
+ } else {
+ result = (HiveCharWritable) previous;
+ }
+ // Use the string reader implementation to populate the internal Text value
+ Object textVal = super.next(result.getTextValue());
+ if (textVal == null) {
+ return null;
+ }
+ // result should now hold the value that was read in.
+ // enforce char length
+ result.enforceMaxLength(maxLength);
+ return result;
+ }
+ }
+
private static class VarcharTreeReader extends StringTreeReader {
int maxLength;
@@ -1890,6 +1919,11 @@ class RecordReaderImpl implements Record
return new LongTreeReader(path, columnId);
case STRING:
return new StringTreeReader(path, columnId);
+ case CHAR:
+ if (!type.hasMaximumLength()) {
+ throw new IllegalArgumentException("ORC char type has no length specified");
+ }
+ return new CharTreeReader(path, columnId, type.getMaximumLength());
case VARCHAR:
if (!type.hasMaximumLength()) {
throw new IllegalArgumentException("ORC varchar type has no length specified");