You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/05 21:15:48 UTC
svn commit: r1622763 [2/17] - in /hive/branches/tez: ./
ant/src/org/apache/hadoop/hive/ant/
hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/
hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/
itests/hive-unit/ itests/hive-un...
Modified: hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java?rev=1622763&r1=1622762&r2=1622763&view=diff
==============================================================================
--- hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java (original)
+++ hive/branches/tez/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java Fri Sep 5 19:15:44 2014
@@ -232,29 +232,98 @@ public class GenVectorCode extends Task
{"FilterScalarCompareColumn", "GreaterEqual", "long", "long", ">="},
{"FilterScalarCompareColumn", "GreaterEqual", "double", "long", ">="},
- {"FilterStringColumnCompareScalar", "Equal", "=="},
- {"FilterStringColumnCompareScalar", "NotEqual", "!="},
- {"FilterStringColumnCompareScalar", "Less", "<"},
- {"FilterStringColumnCompareScalar", "LessEqual", "<="},
- {"FilterStringColumnCompareScalar", "Greater", ">"},
- {"FilterStringColumnCompareScalar", "GreaterEqual", ">="},
+ {"FilterStringGroupColumnCompareStringGroupScalarBase", "Equal", "=="},
+ {"FilterStringGroupColumnCompareStringGroupScalarBase", "NotEqual", "!="},
+ {"FilterStringGroupColumnCompareStringGroupScalarBase", "Less", "<"},
+ {"FilterStringGroupColumnCompareStringGroupScalarBase", "LessEqual", "<="},
+ {"FilterStringGroupColumnCompareStringGroupScalarBase", "Greater", ">"},
+ {"FilterStringGroupColumnCompareStringGroupScalarBase", "GreaterEqual", ">="},
+
+ {"FilterStringGroupColumnCompareStringScalar", "Equal", "=="},
+ {"FilterStringGroupColumnCompareStringScalar", "NotEqual", "!="},
+ {"FilterStringGroupColumnCompareStringScalar", "Less", "<"},
+ {"FilterStringGroupColumnCompareStringScalar", "LessEqual", "<="},
+ {"FilterStringGroupColumnCompareStringScalar", "Greater", ">"},
+ {"FilterStringGroupColumnCompareStringScalar", "GreaterEqual", ">="},
+
+ {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "Equal", "=="},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "NotEqual", "!="},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "Less", "<"},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "LessEqual", "<="},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "Greater", ">"},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "VarChar", "GreaterEqual", ">="},
+
+ {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "Equal", "=="},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "NotEqual", "!="},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "Less", "<"},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "LessEqual", "<="},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "Greater", ">"},
+ {"FilterStringGroupColumnCompareTruncStringScalar", "Char", "GreaterEqual", ">="},
{"FilterStringColumnBetween", ""},
{"FilterStringColumnBetween", "!"},
- {"StringColumnCompareScalar", "Equal", "=="},
- {"StringColumnCompareScalar", "NotEqual", "!="},
- {"StringColumnCompareScalar", "Less", "<"},
- {"StringColumnCompareScalar", "LessEqual", "<="},
- {"StringColumnCompareScalar", "Greater", ">"},
- {"StringColumnCompareScalar", "GreaterEqual", ">="},
-
- {"FilterStringScalarCompareColumn", "Equal", "=="},
- {"FilterStringScalarCompareColumn", "NotEqual", "!="},
- {"FilterStringScalarCompareColumn", "Less", "<"},
- {"FilterStringScalarCompareColumn", "LessEqual", "<="},
- {"FilterStringScalarCompareColumn", "Greater", ">"},
- {"FilterStringScalarCompareColumn", "GreaterEqual", ">="},
+ {"FilterTruncStringColumnBetween", "VarChar", ""},
+ {"FilterTruncStringColumnBetween", "VarChar", "!"},
+
+ {"FilterTruncStringColumnBetween", "Char", ""},
+ {"FilterTruncStringColumnBetween", "Char", "!"},
+
+ {"StringGroupColumnCompareStringGroupScalarBase", "Equal", "=="},
+ {"StringGroupColumnCompareStringGroupScalarBase", "NotEqual", "!="},
+ {"StringGroupColumnCompareStringGroupScalarBase", "Less", "<"},
+ {"StringGroupColumnCompareStringGroupScalarBase", "LessEqual", "<="},
+ {"StringGroupColumnCompareStringGroupScalarBase", "Greater", ">"},
+ {"StringGroupColumnCompareStringGroupScalarBase", "GreaterEqual", ">="},
+
+ {"StringGroupColumnCompareStringScalar", "Equal", "=="},
+ {"StringGroupColumnCompareStringScalar", "NotEqual", "!="},
+ {"StringGroupColumnCompareStringScalar", "Less", "<"},
+ {"StringGroupColumnCompareStringScalar", "LessEqual", "<="},
+ {"StringGroupColumnCompareStringScalar", "Greater", ">"},
+ {"StringGroupColumnCompareStringScalar", "GreaterEqual", ">="},
+
+ {"StringGroupColumnCompareTruncStringScalar", "VarChar", "Equal", "=="},
+ {"StringGroupColumnCompareTruncStringScalar", "VarChar", "NotEqual", "!="},
+ {"StringGroupColumnCompareTruncStringScalar", "VarChar", "Less", "<"},
+ {"StringGroupColumnCompareTruncStringScalar", "VarChar", "LessEqual", "<="},
+ {"StringGroupColumnCompareTruncStringScalar", "VarChar", "Greater", ">"},
+ {"StringGroupColumnCompareTruncStringScalar", "VarChar", "GreaterEqual", ">="},
+
+ {"StringGroupColumnCompareTruncStringScalar", "Char", "Equal", "=="},
+ {"StringGroupColumnCompareTruncStringScalar", "Char", "NotEqual", "!="},
+ {"StringGroupColumnCompareTruncStringScalar", "Char", "Less", "<"},
+ {"StringGroupColumnCompareTruncStringScalar", "Char", "LessEqual", "<="},
+ {"StringGroupColumnCompareTruncStringScalar", "Char", "Greater", ">"},
+ {"StringGroupColumnCompareTruncStringScalar", "Char", "GreaterEqual", ">="},
+
+ {"FilterStringGroupScalarCompareStringGroupColumnBase", "Equal", "=="},
+ {"FilterStringGroupScalarCompareStringGroupColumnBase", "NotEqual", "!="},
+ {"FilterStringGroupScalarCompareStringGroupColumnBase", "Less", "<"},
+ {"FilterStringGroupScalarCompareStringGroupColumnBase", "LessEqual", "<="},
+ {"FilterStringGroupScalarCompareStringGroupColumnBase", "Greater", ">"},
+ {"FilterStringGroupScalarCompareStringGroupColumnBase", "GreaterEqual", ">="},
+
+ {"FilterStringScalarCompareStringGroupColumn", "Equal", "=="},
+ {"FilterStringScalarCompareStringGroupColumn", "NotEqual", "!="},
+ {"FilterStringScalarCompareStringGroupColumn", "Less", "<"},
+ {"FilterStringScalarCompareStringGroupColumn", "LessEqual", "<="},
+ {"FilterStringScalarCompareStringGroupColumn", "Greater", ">"},
+ {"FilterStringScalarCompareStringGroupColumn", "GreaterEqual", ">="},
+
+ {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "Equal", "=="},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "NotEqual", "!="},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "Less", "<"},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "LessEqual", "<="},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "Greater", ">"},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "VarChar", "GreaterEqual", ">="},
+
+ {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "Equal", "=="},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "NotEqual", "!="},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "Less", "<"},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "LessEqual", "<="},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "Greater", ">"},
+ {"FilterTruncStringScalarCompareStringGroupColumn", "Char", "GreaterEqual", ">="},
{"FilterDecimalColumnCompareScalar", "Equal", "=="},
{"FilterDecimalColumnCompareScalar", "NotEqual", "!="},
@@ -277,26 +346,47 @@ public class GenVectorCode extends Task
{"FilterDecimalColumnCompareColumn", "Greater", ">"},
{"FilterDecimalColumnCompareColumn", "GreaterEqual", ">="},
- {"StringScalarCompareColumn", "Equal", "=="},
- {"StringScalarCompareColumn", "NotEqual", "!="},
- {"StringScalarCompareColumn", "Less", "<"},
- {"StringScalarCompareColumn", "LessEqual", "<="},
- {"StringScalarCompareColumn", "Greater", ">"},
- {"StringScalarCompareColumn", "GreaterEqual", ">="},
-
- {"FilterStringColumnCompareColumn", "Equal", "=="},
- {"FilterStringColumnCompareColumn", "NotEqual", "!="},
- {"FilterStringColumnCompareColumn", "Less", "<"},
- {"FilterStringColumnCompareColumn", "LessEqual", "<="},
- {"FilterStringColumnCompareColumn", "Greater", ">"},
- {"FilterStringColumnCompareColumn", "GreaterEqual", ">="},
-
- {"StringColumnCompareColumn", "Equal", "=="},
- {"StringColumnCompareColumn", "NotEqual", "!="},
- {"StringColumnCompareColumn", "Less", "<"},
- {"StringColumnCompareColumn", "LessEqual", "<="},
- {"StringColumnCompareColumn", "Greater", ">"},
- {"StringColumnCompareColumn", "GreaterEqual", ">="},
+ {"StringGroupScalarCompareStringGroupColumnBase", "Equal", "=="},
+ {"StringGroupScalarCompareStringGroupColumnBase", "NotEqual", "!="},
+ {"StringGroupScalarCompareStringGroupColumnBase", "Less", "<"},
+ {"StringGroupScalarCompareStringGroupColumnBase", "LessEqual", "<="},
+ {"StringGroupScalarCompareStringGroupColumnBase", "Greater", ">"},
+ {"StringGroupScalarCompareStringGroupColumnBase", "GreaterEqual", ">="},
+
+ {"StringScalarCompareStringGroupColumn", "Equal", "=="},
+ {"StringScalarCompareStringGroupColumn", "NotEqual", "!="},
+ {"StringScalarCompareStringGroupColumn", "Less", "<"},
+ {"StringScalarCompareStringGroupColumn", "LessEqual", "<="},
+ {"StringScalarCompareStringGroupColumn", "Greater", ">"},
+ {"StringScalarCompareStringGroupColumn", "GreaterEqual", ">="},
+
+ {"TruncStringScalarCompareStringGroupColumn", "VarChar", "Equal", "=="},
+ {"TruncStringScalarCompareStringGroupColumn", "VarChar", "NotEqual", "!="},
+ {"TruncStringScalarCompareStringGroupColumn", "VarChar", "Less", "<"},
+ {"TruncStringScalarCompareStringGroupColumn", "VarChar", "LessEqual", "<="},
+ {"TruncStringScalarCompareStringGroupColumn", "VarChar", "Greater", ">"},
+ {"TruncStringScalarCompareStringGroupColumn", "VarChar", "GreaterEqual", ">="},
+
+ {"TruncStringScalarCompareStringGroupColumn", "Char", "Equal", "=="},
+ {"TruncStringScalarCompareStringGroupColumn", "Char", "NotEqual", "!="},
+ {"TruncStringScalarCompareStringGroupColumn", "Char", "Less", "<"},
+ {"TruncStringScalarCompareStringGroupColumn", "Char", "LessEqual", "<="},
+ {"TruncStringScalarCompareStringGroupColumn", "Char", "Greater", ">"},
+ {"TruncStringScalarCompareStringGroupColumn", "Char", "GreaterEqual", ">="},
+
+ {"FilterStringGroupColumnCompareStringGroupColumn", "Equal", "=="},
+ {"FilterStringGroupColumnCompareStringGroupColumn", "NotEqual", "!="},
+ {"FilterStringGroupColumnCompareStringGroupColumn", "Less", "<"},
+ {"FilterStringGroupColumnCompareStringGroupColumn", "LessEqual", "<="},
+ {"FilterStringGroupColumnCompareStringGroupColumn", "Greater", ">"},
+ {"FilterStringGroupColumnCompareStringGroupColumn", "GreaterEqual", ">="},
+
+ {"StringGroupColumnCompareStringGroupColumn", "Equal", "=="},
+ {"StringGroupColumnCompareStringGroupColumn", "NotEqual", "!="},
+ {"StringGroupColumnCompareStringGroupColumn", "Less", "<"},
+ {"StringGroupColumnCompareStringGroupColumn", "LessEqual", "<="},
+ {"StringGroupColumnCompareStringGroupColumn", "Greater", ">"},
+ {"StringGroupColumnCompareStringGroupColumn", "GreaterEqual", ">="},
{"FilterColumnCompareColumn", "Equal", "long", "double", "=="},
{"FilterColumnCompareColumn", "Equal", "double", "double", "=="},
@@ -359,58 +449,58 @@ public class GenVectorCode extends Task
{"ColumnCompareColumn", "GreaterEqual", "double", "long", ">="},
// template, <ClassNamePrefix>, <ReturnType>, <OperandType>, <FuncName>, <OperandCast>,
- // <ResultCast>, <Cleanup>
- {"ColumnUnaryFunc", "FuncRound", "double", "double", "MathExpr.round", "", "", ""},
+ // <ResultCast>, <Cleanup> <VectorExprArgType>
+ {"ColumnUnaryFunc", "FuncRound", "double", "double", "MathExpr.round", "", "", "", ""},
// round(longCol) returns a long and is a no-op. So it will not be implemented here.
// round(Col, N) is a special case and will be implemented separately from this template
- {"ColumnUnaryFunc", "FuncFloor", "long", "double", "Math.floor", "", "(long)", ""},
+ {"ColumnUnaryFunc", "FuncFloor", "long", "double", "Math.floor", "", "(long)", "", ""},
// Floor on an integer argument is a noop, but it is less code to handle it this way.
- {"ColumnUnaryFunc", "FuncFloor", "long", "long", "Math.floor", "", "(long)", ""},
- {"ColumnUnaryFunc", "FuncCeil", "long", "double", "Math.ceil", "", "(long)", ""},
+ {"ColumnUnaryFunc", "FuncFloor", "long", "long", "Math.floor", "", "(long)", "", ""},
+ {"ColumnUnaryFunc", "FuncCeil", "long", "double", "Math.ceil", "", "(long)", "", ""},
// Ceil on an integer argument is a noop, but it is less code to handle it this way.
- {"ColumnUnaryFunc", "FuncCeil", "long", "long", "Math.ceil", "", "(long)", ""},
- {"ColumnUnaryFunc", "FuncExp", "double", "double", "Math.exp", "", "", ""},
- {"ColumnUnaryFunc", "FuncExp", "double", "long", "Math.exp", "(double)", "", ""},
+ {"ColumnUnaryFunc", "FuncCeil", "long", "long", "Math.ceil", "", "(long)", "", ""},
+ {"ColumnUnaryFunc", "FuncExp", "double", "double", "Math.exp", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncExp", "double", "long", "Math.exp", "(double)", "", "", ""},
{"ColumnUnaryFunc", "FuncLn", "double", "double", "Math.log", "", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""},
{"ColumnUnaryFunc", "FuncLn", "double", "long", "Math.log", "(double)", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""},
{"ColumnUnaryFunc", "FuncLog10", "double", "double", "Math.log10", "", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""},
{"ColumnUnaryFunc", "FuncLog10", "double", "long", "Math.log10", "(double)", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""},
// The MathExpr class contains helper functions for cases when existing library
// routines can't be used directly.
{"ColumnUnaryFunc", "FuncLog2", "double", "double", "MathExpr.log2", "", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""},
{"ColumnUnaryFunc", "FuncLog2", "double", "long", "MathExpr.log2", "(double)", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);"},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n, true);", ""},
// Log(base, Col) is a special case and will be implemented separately from this template
// Pow(col, P) and Power(col, P) are special cases implemented separately from this template
{"ColumnUnaryFunc", "FuncSqrt", "double", "double", "Math.sqrt", "", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);"},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);", ""},
{"ColumnUnaryFunc", "FuncSqrt", "double", "long", "Math.sqrt", "(double)", "",
- "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);"},
- {"ColumnUnaryFunc", "FuncAbs", "double", "double", "Math.abs", "", "", ""},
- {"ColumnUnaryFunc", "FuncAbs", "long", "long", "MathExpr.abs", "", "", ""},
- {"ColumnUnaryFunc", "FuncSin", "double", "double", "Math.sin", "", "", ""},
- {"ColumnUnaryFunc", "FuncSin", "double", "long", "Math.sin", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncASin", "double", "double", "Math.asin", "", "", ""},
- {"ColumnUnaryFunc", "FuncASin", "double", "long", "Math.asin", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncCos", "double", "double", "Math.cos", "", "", ""},
- {"ColumnUnaryFunc", "FuncCos", "double", "long", "Math.cos", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncACos", "double", "double", "Math.acos", "", "", ""},
- {"ColumnUnaryFunc", "FuncACos", "double", "long", "Math.acos", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncTan", "double", "double", "Math.tan", "", "", ""},
- {"ColumnUnaryFunc", "FuncTan", "double", "long", "Math.tan", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncATan", "double", "double", "Math.atan", "", "", ""},
- {"ColumnUnaryFunc", "FuncATan", "double", "long", "Math.atan", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncDegrees", "double", "double", "Math.toDegrees", "", "", ""},
- {"ColumnUnaryFunc", "FuncDegrees", "double", "long", "Math.toDegrees", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncRadians", "double", "double", "Math.toRadians", "", "", ""},
- {"ColumnUnaryFunc", "FuncRadians", "double", "long", "Math.toRadians", "(double)", "", ""},
- {"ColumnUnaryFunc", "FuncSign", "double", "double", "MathExpr.sign", "", "", ""},
- {"ColumnUnaryFunc", "FuncSign", "double", "long", "MathExpr.sign", "(double)", "", ""},
+ "MathExpr.NaNToNull(outputColVector, sel, batch.selectedInUse, n);", ""},
+ {"ColumnUnaryFunc", "FuncAbs", "double", "double", "Math.abs", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncAbs", "long", "long", "MathExpr.abs", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncSin", "double", "double", "Math.sin", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncSin", "double", "long", "Math.sin", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncASin", "double", "double", "Math.asin", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncASin", "double", "long", "Math.asin", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncCos", "double", "double", "Math.cos", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncCos", "double", "long", "Math.cos", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncACos", "double", "double", "Math.acos", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncACos", "double", "long", "Math.acos", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncTan", "double", "double", "Math.tan", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncTan", "double", "long", "Math.tan", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncATan", "double", "double", "Math.atan", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncATan", "double", "long", "Math.atan", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncDegrees", "double", "double", "Math.toDegrees", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncDegrees", "double", "long", "Math.toDegrees", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncRadians", "double", "double", "Math.toRadians", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncRadians", "double", "long", "Math.toRadians", "(double)", "", "", ""},
+ {"ColumnUnaryFunc", "FuncSign", "double", "double", "MathExpr.sign", "", "", "", ""},
+ {"ColumnUnaryFunc", "FuncSign", "double", "long", "MathExpr.sign", "(double)", "", "", ""},
{"DecimalColumnUnaryFunc", "FuncFloor", "decimal", "DecimalUtil.floor"},
{"DecimalColumnUnaryFunc", "FuncCeil", "decimal", "DecimalUtil.ceiling"},
@@ -420,20 +510,24 @@ public class GenVectorCode extends Task
{"DecimalColumnUnaryFunc", "FuncNegate", "decimal", "DecimalUtil.negate"},
// Casts
- {"ColumnUnaryFunc", "Cast", "long", "double", "", "", "(long)", ""},
- {"ColumnUnaryFunc", "Cast", "double", "long", "", "", "(double)", ""},
+ {"ColumnUnaryFunc", "Cast", "long", "double", "", "", "(long)", "", ""},
+ {"ColumnUnaryFunc", "Cast", "double", "long", "", "", "(double)", "", ""},
{"ColumnUnaryFunc", "CastTimestampToLongVia", "long", "long", "MathExpr.fromTimestamp", "",
- "", ""},
+ "", "", "timestamp"},
{"ColumnUnaryFunc", "CastTimestampToDoubleVia", "double", "long",
- "MathExpr.fromTimestampToDouble", "", "", ""},
+ "MathExpr.fromTimestampToDouble", "", "", "", "timestamp"},
{"ColumnUnaryFunc", "CastDoubleToBooleanVia", "long", "double", "MathExpr.toBool", "",
- "", ""},
+ "", "", ""},
{"ColumnUnaryFunc", "CastLongToBooleanVia", "long", "long", "MathExpr.toBool", "",
- "", ""},
+ "", "", ""},
+ {"ColumnUnaryFunc", "CastDateToBooleanVia", "long", "long", "MathExpr.toBool", "",
+ "", "", "date"},
+ {"ColumnUnaryFunc", "CastTimestampToBooleanVia", "long", "long", "MathExpr.toBool", "",
+ "", "", "timestamp"},
{"ColumnUnaryFunc", "CastLongToTimestampVia", "long", "long", "MathExpr.longToTimestamp", "",
- "", ""},
+ "", "", ""},
{"ColumnUnaryFunc", "CastDoubleToTimestampVia", "long", "double",
- "MathExpr.doubleToTimestamp", "", "", ""},
+ "MathExpr.doubleToTimestamp", "", "", "", ""},
// Boolean to long is done with an IdentityExpression
// Boolean to double is done with standard Long to Double cast
@@ -658,22 +752,40 @@ public class GenVectorCode extends Task
generateVectorUDAFVar(tdesc);
} else if (tdesc[0].equals("VectorUDAFVarDecimal")) {
generateVectorUDAFVarDecimal(tdesc);
- } else if (tdesc[0].equals("FilterStringColumnCompareScalar")) {
- generateFilterStringColumnCompareScalar(tdesc);
+ } else if (tdesc[0].equals("FilterStringGroupColumnCompareStringGroupScalarBase")) {
+ generateFilterStringGroupColumnCompareStringGroupScalarBase(tdesc);
+ } else if (tdesc[0].equals("FilterStringGroupColumnCompareStringScalar")) {
+ generateFilterStringGroupColumnCompareStringScalar(tdesc);
+ } else if (tdesc[0].equals("FilterStringGroupColumnCompareTruncStringScalar")) {
+ generateFilterStringGroupColumnCompareTruncStringScalar(tdesc);
} else if (tdesc[0].equals("FilterStringColumnBetween")) {
generateFilterStringColumnBetween(tdesc);
+ } else if (tdesc[0].equals("FilterTruncStringColumnBetween")) {
+ generateFilterTruncStringColumnBetween(tdesc);
} else if (tdesc[0].equals("FilterDecimalColumnBetween")) {
generateFilterDecimalColumnBetween(tdesc);
- } else if (tdesc[0].equals("StringColumnCompareScalar")) {
- generateStringColumnCompareScalar(tdesc);
- } else if (tdesc[0].equals("FilterStringScalarCompareColumn")) {
- generateFilterStringScalarCompareColumn(tdesc);
- } else if (tdesc[0].equals("StringScalarCompareColumn")) {
- generateStringScalarCompareColumn(tdesc);
- } else if (tdesc[0].equals("FilterStringColumnCompareColumn")) {
- generateFilterStringColumnCompareColumn(tdesc);
- } else if (tdesc[0].equals("StringColumnCompareColumn")) {
- generateStringColumnCompareColumn(tdesc);
+ } else if (tdesc[0].equals("StringGroupColumnCompareStringGroupScalarBase")) {
+ generateStringGroupColumnCompareStringGroupScalarBase(tdesc);
+ } else if (tdesc[0].equals("StringGroupColumnCompareStringScalar")) {
+ generateStringGroupColumnCompareStringScalar(tdesc);
+ } else if (tdesc[0].equals("StringGroupColumnCompareTruncStringScalar")) {
+ generateStringGroupColumnCompareTruncStringScalar(tdesc);
+ } else if (tdesc[0].equals("FilterStringGroupScalarCompareStringGroupColumnBase")) {
+ generateFilterStringGroupScalarCompareStringGroupColumnBase(tdesc);
+ } else if (tdesc[0].equals("FilterStringScalarCompareStringGroupColumn")) {
+ generateFilterStringScalarCompareStringGroupColumn(tdesc);
+ } else if (tdesc[0].equals("FilterTruncStringScalarCompareStringGroupColumn")) {
+ generateFilterTruncStringScalarCompareStringGroupColumn(tdesc);
+ } else if (tdesc[0].equals("StringGroupScalarCompareStringGroupColumnBase")) {
+ generateStringGroupScalarCompareStringGroupColumnBase(tdesc);
+ } else if (tdesc[0].equals("StringScalarCompareStringGroupColumn")) {
+ generateStringScalarCompareStringGroupColumn(tdesc);
+ } else if (tdesc[0].equals("TruncStringScalarCompareStringGroupColumn")) {
+ generateTruncStringScalarCompareStringGroupColumn(tdesc);
+ } else if (tdesc[0].equals("FilterStringGroupColumnCompareStringGroupColumn")) {
+ generateFilterStringGroupColumnCompareStringGroupColumn(tdesc);
+ } else if (tdesc[0].equals("StringGroupColumnCompareStringGroupColumn")) {
+ generateStringGroupColumnCompareStringGroupColumn(tdesc);
} else if (tdesc[0].equals("IfExprColumnColumn")) {
generateIfExprColumnColumn(tdesc);
} else if (tdesc[0].equals("IfExprColumnScalar")) {
@@ -710,6 +822,35 @@ public class GenVectorCode extends Task
className, templateString);
}
+ private void generateFilterTruncStringColumnBetween(String[] tdesc) throws IOException {
+ String truncStringTypeName = tdesc[1];
+ String truncStringHiveType;
+ String truncStringHiveGetBytes;
+ if (truncStringTypeName == "Char") {
+ truncStringHiveType = "HiveChar";
+ truncStringHiveGetBytes = "getStrippedValue().getBytes()";
+ } else if (truncStringTypeName == "VarChar") {
+ truncStringHiveType = "HiveVarchar";
+ truncStringHiveGetBytes = "getValue().getBytes()";
+ } else {
+ throw new Error("Unsupported string type: " + truncStringTypeName);
+ }
+ String optionalNot = tdesc[2];
+ String className = "Filter" + truncStringTypeName + "Column" + (optionalNot.equals("!") ? "Not" : "")
+ + "Between";
+ // Read the template into a string, expand it, and write it.
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<TruncStringTypeName>", truncStringTypeName);
+ templateString = templateString.replaceAll("<TruncStringHiveType>", truncStringHiveType);
+ templateString = templateString.replaceAll("<TruncStringHiveGetBytes>", truncStringHiveGetBytes);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<OptionalNot>", optionalNot);
+
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
private void generateFilterDecimalColumnBetween(String[] tdesc) throws IOException {
String optionalNot = tdesc[1];
String className = "FilterDecimalColumn" + (optionalNot.equals("!") ? "Not" : "")
@@ -745,14 +886,12 @@ public class GenVectorCode extends Task
}
private void generateColumnCompareColumn(String[] tdesc) throws Exception {
- //The variables are all same as ColumnCompareScalar except that
- //this template doesn't need a return type. Pass anything as return type.
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
String className = getCamelCaseType(operandType1)
+ "Col" + operatorName + getCamelCaseType(operandType2) + "Column";
- generateColumnBinaryOperatorColumn(tdesc, "long", className);
+ generateColumnCompareOperatorColumn(tdesc, false, className);
}
private void generateVectorUDAFMinMax(String[] tdesc) throws Exception {
@@ -886,45 +1025,147 @@ public class GenVectorCode extends Task
writeFile(templateFile.lastModified(), udafOutputDirectory, udafClassesDirectory,
className, templateString);
}
-
-
- private void generateFilterStringScalarCompareColumn(String[] tdesc) throws IOException {
+
+ private void generateFilterStringGroupScalarCompareStringGroupColumnBase(String[] tdesc) throws IOException {
String operatorName = tdesc[1];
- String className = "FilterStringScalar" + operatorName + "StringColumn";
+ String className = "FilterStringGroupScalar" + operatorName + "StringGroupColumnBase";
// Template expansion logic is the same for both column-scalar and scalar-column cases.
generateStringColumnCompareScalar(tdesc, className);
}
- private void generateStringScalarCompareColumn(String[] tdesc) throws IOException {
+ private void generateFilterStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException {
+ String operatorName = tdesc[1];
+ String className = "FilterStringScalar" + operatorName + "StringGroupColumn";
+ String baseClassName = "FilterStringGroupScalar" + operatorName + "StringGroupColumnBase";
+ String operatorSymbol = tdesc[2];
+ // Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateFilterTruncStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException {
+ String truncStringTypeName = tdesc[1];
+ String operatorName = tdesc[2];
+ String className = "Filter" + truncStringTypeName + "Scalar" + operatorName + "StringGroupColumn";
+ String baseClassName = "FilterStringGroupScalar" + operatorName + "StringGroupColumnBase";
+ generateStringCompareTruncStringScalar(tdesc, className, baseClassName);
+ }
+
+ private void generateStringGroupScalarCompareStringGroupColumnBase(String[] tdesc) throws IOException {
String operatorName = tdesc[1];
- String className = "StringScalar" + operatorName + "StringColumn";
+ String className = "StringGroupScalar" + operatorName + "StringGroupColumnBase";
// Template expansion logic is the same for both column-scalar and scalar-column cases.
generateStringColumnCompareScalar(tdesc, className);
}
- private void generateFilterStringColumnCompareScalar(String[] tdesc) throws IOException {
+ private void generateStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException {
String operatorName = tdesc[1];
- String className = "FilterStringCol" + operatorName + "StringScalar";
- generateStringColumnCompareScalar(tdesc, className);
+ String className = "StringScalar" + operatorName + "StringGroupColumn";
+ String baseClassName = "StringGroupScalar" + operatorName + "StringGroupColumnBase";
+ String operatorSymbol = tdesc[2];
+ // Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString); }
+
+ private void generateTruncStringScalarCompareStringGroupColumn(String[] tdesc) throws IOException {
+ String truncStringTypeName = tdesc[1];
+ String operatorName = tdesc[2];
+ String className = truncStringTypeName + "Scalar" + operatorName + "StringGroupColumn";
+ String baseClassName = "StringGroupScalar" + operatorName + "StringGroupColumnBase";
+ generateStringCompareTruncStringScalar(tdesc, className, baseClassName);
}
- private void generateStringColumnCompareScalar(String[] tdesc) throws IOException {
+ private void generateFilterStringGroupColumnCompareStringGroupScalarBase(String[] tdesc) throws IOException {
String operatorName = tdesc[1];
- String className = "StringCol" + operatorName + "StringScalar";
+ String className = "FilterStringGroupCol" + operatorName + "StringGroupScalarBase";
generateStringColumnCompareScalar(tdesc, className);
}
- private void generateFilterStringColumnCompareColumn(String[] tdesc) throws IOException {
+ private void generateFilterStringGroupColumnCompareStringScalar(String[] tdesc) throws IOException {
+ String operatorName = tdesc[1];
+ String className = "FilterStringGroupCol" + operatorName + "StringScalar";
+ String baseClassName = "FilterStringGroupCol" + operatorName + "StringGroupScalarBase";
+ String operatorSymbol = tdesc[2];
+ // Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateFilterStringGroupColumnCompareTruncStringScalar(String[] tdesc) throws IOException {
+ String truncStringTypeName = tdesc[1];
+ String operatorName = tdesc[2];
+ String className = "FilterStringGroupCol" + operatorName + truncStringTypeName + "Scalar";
+ String baseClassName = "FilterStringGroupCol" + operatorName + "StringGroupScalarBase";
+ generateStringCompareTruncStringScalar(tdesc, className, baseClassName);
+ }
+
+ private void generateStringGroupColumnCompareStringGroupScalarBase(String[] tdesc) throws IOException {
+ String operatorName = tdesc[1];
+ String className = "StringGroupCol" + operatorName + "StringGroupScalarBase";
+ String operatorSymbol = tdesc[2];
+ // Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateStringGroupColumnCompareStringScalar(String[] tdesc) throws IOException {
+ String operatorName = tdesc[1];
+ String className = "StringGroupCol" + operatorName + "StringScalar";
+ String baseClassName = "StringGroupCol" + operatorName + "StringGroupScalarBase";
+ String operatorSymbol = tdesc[2];
+ // Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
+ private void generateStringGroupColumnCompareTruncStringScalar(String[] tdesc) throws IOException {
+ String truncStringTypeName = tdesc[1];
+ String operatorName = tdesc[2];
+ String className = "StringGroupCol" + operatorName + truncStringTypeName + "Scalar";
+ String baseClassName = "StringGroupCol" + operatorName + "StringGroupScalarBase";
+ generateStringCompareTruncStringScalar(tdesc, className, baseClassName);
+ }
+
+ private void generateFilterStringGroupColumnCompareStringGroupColumn(String[] tdesc) throws IOException {
String operatorName = tdesc[1];
- String className = "FilterStringCol" + operatorName + "StringColumn";
+ String className = "FilterStringGroupCol" + operatorName + "StringGroupColumn";
generateStringColumnCompareScalar(tdesc, className);
}
- private void generateStringColumnCompareColumn(String[] tdesc) throws IOException {
+ private void generateStringGroupColumnCompareStringGroupColumn(String[] tdesc) throws IOException {
String operatorName = tdesc[1];
- String className = "StringCol" + operatorName + "StringColumn";
+ String className = "StringGroupCol" + operatorName + "StringGroupColumn";
generateStringColumnCompareScalar(tdesc, className);
}
@@ -941,15 +1182,42 @@ public class GenVectorCode extends Task
className, templateString);
}
+ private void generateStringCompareTruncStringScalar(String[] tdesc, String className, String baseClassName)
+ throws IOException {
+ String truncStringTypeName = tdesc[1];
+ String truncStringHiveType;
+ String truncStringHiveGetBytes;
+ if (truncStringTypeName == "Char") {
+ truncStringHiveType = "HiveChar";
+ truncStringHiveGetBytes = "getStrippedValue().getBytes()";
+ } else if (truncStringTypeName == "VarChar") {
+ truncStringHiveType = "HiveVarchar";
+ truncStringHiveGetBytes = "getValue().getBytes()";
+ } else {
+ throw new Error("Unsupported string type: " + truncStringTypeName);
+ }
+ String operatorSymbol = tdesc[3];
+ // Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ // Expand, and write result
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<BaseClassName>", baseClassName);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ templateString = templateString.replaceAll("<TruncStringTypeName>", truncStringTypeName);
+ templateString = templateString.replaceAll("<TruncStringHiveType>", truncStringHiveType);
+ templateString = templateString.replaceAll("<TruncStringHiveGetBytes>", truncStringHiveGetBytes);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+ }
+
private void generateFilterColumnCompareColumn(String[] tdesc) throws Exception {
- //The variables are all same as ColumnCompareScalar except that
- //this template doesn't need a return type. Pass anything as return type.
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
String className = "Filter" + getCamelCaseType(operandType1)
+ "Col" + operatorName + getCamelCaseType(operandType2) + "Column";
- generateColumnBinaryOperatorColumn(tdesc, null, className);
+ generateColumnCompareOperatorColumn(tdesc, true, className);
}
private void generateColumnUnaryMinus(String[] tdesc) throws Exception {
@@ -984,6 +1252,13 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<ClassName>", className);
templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
templateString = templateString.replaceAll("<OperandType>", operandType);
+ String vectorExprArgType = operandType;
+ if (operandType.equals("long")) {
+ // Let comparisons occur for DATE and TIMESTAMP, too.
+ vectorExprArgType = "int_datetime_family";
+ }
+ templateString = templateString.replaceAll("<VectorExprArgType>", vectorExprArgType);
+
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
}
@@ -1006,6 +1281,15 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<OperandType3>", operandType3);
templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ String vectorExprArgType2 = operandType2;
+ String vectorExprArgType3 = operandType3;
+ if (operandType2.equals("long") && operandType3.equals("long")) {
+ // Let this work occur for DATE and TIMESTAMP, too.
+ vectorExprArgType2 = "int_datetime_family";
+ vectorExprArgType3 = "int_datetime_family";
+ }
+ templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
+ templateString = templateString.replaceAll("<VectorExprArgType3>", vectorExprArgType3);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
}
@@ -1028,6 +1312,15 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<OperandType3>", operandType3);
templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ String vectorExprArgType2 = operandType2;
+ String vectorExprArgType3 = operandType3;
+ if (operandType2.equals("long") && operandType3.equals("long")) {
+ // Let this work occur for DATE and TIMESTAMP, too.
+ vectorExprArgType2 = "int_datetime_family";
+ vectorExprArgType3 = "int_datetime_family";
+ }
+ templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
+ templateString = templateString.replaceAll("<VectorExprArgType3>", vectorExprArgType3);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
}
@@ -1049,6 +1342,15 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<OperandType3>", operandType3);
templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ String vectorExprArgType2 = operandType2;
+ String vectorExprArgType3 = operandType3;
+ if (operandType2.equals("long") && operandType3.equals("long")) {
+ // Let this work occur for DATE and TIMESTAMP, too.
+ vectorExprArgType2 = "int_datetime_family";
+ vectorExprArgType3 = "int_datetime_family";
+ }
+ templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
+ templateString = templateString.replaceAll("<VectorExprArgType3>", vectorExprArgType3);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
}
@@ -1073,6 +1375,7 @@ public class GenVectorCode extends Task
}
// template, <ClassNamePrefix>, <ReturnType>, <OperandType>, <FuncName>, <OperandCast>, <ResultCast>
+ // <Cleanup> <VectorExprArgType>
private void generateColumnUnaryFunc(String[] tdesc) throws Exception {
String classNamePrefix = tdesc[1];
String operandType = tdesc[3];
@@ -1087,6 +1390,8 @@ public class GenVectorCode extends Task
String operandCast = tdesc[5];
String resultCast = tdesc[6];
String cleanup = tdesc[7];
+ String vectorExprArgType = tdesc[8].isEmpty() ? operandType : tdesc[8];
+
// Expand, and write result
templateString = templateString.replaceAll("<ClassName>", className);
templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
@@ -1097,6 +1402,7 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandCast>", operandCast);
templateString = templateString.replaceAll("<ResultCast>", resultCast);
templateString = templateString.replaceAll("<Cleanup>", cleanup);
+ templateString = templateString.replaceAll("<VectorExprArgType>", vectorExprArgType);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
}
@@ -1108,58 +1414,53 @@ public class GenVectorCode extends Task
String className = getCamelCaseType(operandType1)
+ "Col" + operatorName + getCamelCaseType(operandType2) + "Column";
String returnType = getArithmeticReturnType(operandType1, operandType2);
- generateColumnBinaryOperatorColumn(tdesc, returnType, className);
+ generateColumnArithmeticOperatorColumn(tdesc, returnType, className);
}
private void generateFilterColumnCompareScalar(String[] tdesc) throws Exception {
- //The variables are all same as ColumnCompareScalar except that
- //this template doesn't need a return type. Pass anything as return type.
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
String className = "Filter" + getCamelCaseType(operandType1)
+ "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
- generateColumnBinaryOperatorScalar(tdesc, null, className);
+ generateColumnCompareOperatorScalar(tdesc, true, className);
}
private void generateFilterScalarCompareColumn(String[] tdesc) throws Exception {
- //this template doesn't need a return type. Pass anything as return type.
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
String className = "Filter" + getCamelCaseType(operandType1)
+ "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column";
- generateScalarBinaryOperatorColumn(tdesc, null, className);
+ generateScalarCompareOperatorColumn(tdesc, true, className);
}
private void generateColumnCompareScalar(String[] tdesc) throws Exception {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String returnType = "long";
String className = getCamelCaseType(operandType1)
+ "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
- generateColumnBinaryOperatorScalar(tdesc, returnType, className);
+ generateColumnCompareOperatorScalar(tdesc, false, className);
}
private void generateScalarCompareColumn(String[] tdesc) throws Exception {
String operatorName = tdesc[1];
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String returnType = "long";
String className = getCamelCaseType(operandType1)
+ "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column";
- generateScalarBinaryOperatorColumn(tdesc, returnType, className);
+ generateScalarCompareOperatorColumn(tdesc, false, className);
}
- private void generateColumnBinaryOperatorColumn(String[] tdesc, String returnType,
+ private void generateColumnCompareOperatorColumn(String[] tdesc, boolean filter,
String className) throws Exception {
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String outputColumnVectorType = this.getColumnVectorType(
- returnType == null ? "long" : returnType);
String inputColumnVectorType1 = this.getColumnVectorType(operandType1);
String inputColumnVectorType2 = this.getColumnVectorType(operandType2);
+ String returnType = "long";
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
String operatorSymbol = tdesc[4];
//Read the template into a string;
@@ -1174,16 +1475,25 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ String vectorExprArgType1 = operandType1;
+ String vectorExprArgType2 = operandType2;
+ if (operandType1.equals("long") && operandType2.equals("long")) {
+ // Let comparisons occur for DATE and TIMESTAMP, too.
+ vectorExprArgType1 = "int_datetime_family";
+ vectorExprArgType2 = "int_datetime_family";
+ }
+ templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1);
+ templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
- if(returnType==null){
+ if (filter) {
testCodeGen.addColumnColumnFilterTestCases(
className,
inputColumnVectorType1,
inputColumnVectorType2,
operatorSymbol);
- }else{
+ } else {
testCodeGen.addColumnColumnOperationTestCases(
className,
inputColumnVectorType1,
@@ -1192,13 +1502,44 @@ public class GenVectorCode extends Task
}
}
- private void generateColumnBinaryOperatorScalar(String[] tdesc, String returnType,
+ private void generateColumnArithmeticOperatorColumn(String[] tdesc, String returnType,
+ String className) throws Exception {
+ String operandType1 = tdesc[2];
+ String operandType2 = tdesc[3];
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
+ String inputColumnVectorType1 = this.getColumnVectorType(operandType1);
+ String inputColumnVectorType2 = this.getColumnVectorType(operandType2);
+ String operatorSymbol = tdesc[4];
+
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<InputColumnVectorType1>", inputColumnVectorType1);
+ templateString = templateString.replaceAll("<InputColumnVectorType2>", inputColumnVectorType2);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ templateString = templateString.replaceAll("<OperandType1>", operandType1);
+ templateString = templateString.replaceAll("<OperandType2>", operandType2);
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+ templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+
+ testCodeGen.addColumnColumnOperationTestCases(
+ className,
+ inputColumnVectorType1,
+ inputColumnVectorType2,
+ outputColumnVectorType);
+ }
+
+ private void generateColumnCompareOperatorScalar(String[] tdesc, boolean filter,
String className) throws Exception {
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String outputColumnVectorType = this.getColumnVectorType(
- returnType == null ? "long" : returnType);
String inputColumnVectorType = this.getColumnVectorType(operandType1);
+ String returnType = "long";
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
String operatorSymbol = tdesc[4];
//Read the template into a string;
@@ -1211,10 +1552,20 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType1>", operandType1);
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
+ templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ String vectorExprArgType1 = operandType1;
+ String vectorExprArgType2 = operandType2;
+ if (operandType1.equals("long") && operandType2.equals("long")) {
+ // Let comparisons occur for DATE and TIMESTAMP, too.
+ vectorExprArgType1 = "int_datetime_family";
+ vectorExprArgType2 = "int_datetime_family";
+ }
+ templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1);
+ templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
- if(returnType==null) {
+ if (filter) {
testCodeGen.addColumnScalarFilterTestCases(
true,
className,
@@ -1231,13 +1582,42 @@ public class GenVectorCode extends Task
}
}
- private void generateScalarBinaryOperatorColumn(String[] tdesc, String returnType,
+ private void generateColumnArithmeticOperatorScalar(String[] tdesc, String returnType,
+ String className) throws Exception {
+ String operandType1 = tdesc[2];
+ String operandType2 = tdesc[3];
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
+ String inputColumnVectorType = this.getColumnVectorType(operandType1);
+ String operatorSymbol = tdesc[4];
+
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ templateString = templateString.replaceAll("<OperandType1>", operandType1);
+ templateString = templateString.replaceAll("<OperandType2>", operandType2);
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+
+ testCodeGen.addColumnScalarOperationTestCases(
+ true,
+ className,
+ inputColumnVectorType,
+ outputColumnVectorType,
+ operandType2);
+ }
+
+ private void generateScalarCompareOperatorColumn(String[] tdesc, boolean filter,
String className) throws Exception {
String operandType1 = tdesc[2];
String operandType2 = tdesc[3];
- String outputColumnVectorType = this.getColumnVectorType(
- returnType == null ? "long" : returnType);
+ String returnType = "long";
String inputColumnVectorType = this.getColumnVectorType(operandType2);
+ String outputColumnVectorType = this.getColumnVectorType(returnType);
String operatorSymbol = tdesc[4];
//Read the template into a string;
@@ -1251,10 +1631,19 @@ public class GenVectorCode extends Task
templateString = templateString.replaceAll("<OperandType2>", operandType2);
templateString = templateString.replaceAll("<ReturnType>", returnType);
templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ String vectorExprArgType1 = operandType1;
+ String vectorExprArgType2 = operandType2;
+ if (operandType1.equals("long") && operandType2.equals("long")) {
+ // Let comparisons occur for DATE and TIMESTAMP, too.
+ vectorExprArgType1 = "int_datetime_family";
+ vectorExprArgType2 = "int_datetime_family";
+ }
+ templateString = templateString.replaceAll("<VectorExprArgType1>", vectorExprArgType1);
+ templateString = templateString.replaceAll("<VectorExprArgType2>", vectorExprArgType2);
writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
className, templateString);
- if(returnType==null) {
+ if (filter) {
testCodeGen.addColumnScalarFilterTestCases(
false,
className,
@@ -1269,7 +1658,38 @@ public class GenVectorCode extends Task
outputColumnVectorType,
operandType1);
}
- }
+ }
+
+ private void generateScalarArithmeticOperatorColumn(String[] tdesc, String returnType,
+ String className) throws Exception {
+ String operandType1 = tdesc[2];
+ String operandType2 = tdesc[3];
+ String outputColumnVectorType = this.getColumnVectorType(
+ returnType == null ? "long" : returnType);
+ String inputColumnVectorType = this.getColumnVectorType(operandType2);
+ String operatorSymbol = tdesc[4];
+
+ //Read the template into a string;
+ File templateFile = new File(joinPath(this.expressionTemplateDirectory, tdesc[0] + ".txt"));
+ String templateString = readFile(templateFile);
+ templateString = templateString.replaceAll("<ClassName>", className);
+ templateString = templateString.replaceAll("<InputColumnVectorType>", inputColumnVectorType);
+ templateString = templateString.replaceAll("<OutputColumnVectorType>", outputColumnVectorType);
+ templateString = templateString.replaceAll("<OperatorSymbol>", operatorSymbol);
+ templateString = templateString.replaceAll("<OperandType1>", operandType1);
+ templateString = templateString.replaceAll("<OperandType2>", operandType2);
+ templateString = templateString.replaceAll("<ReturnType>", returnType);
+ templateString = templateString.replaceAll("<CamelReturnType>", getCamelCaseType(returnType));
+ writeFile(templateFile.lastModified(), expressionOutputDirectory, expressionClassesDirectory,
+ className, templateString);
+
+ testCodeGen.addColumnScalarOperationTestCases(
+ false,
+ className,
+ inputColumnVectorType,
+ outputColumnVectorType,
+ operandType1);
+ }
//Binary arithmetic operator
private void generateColumnArithmeticScalar(String[] tdesc) throws Exception {
@@ -1279,7 +1699,7 @@ public class GenVectorCode extends Task
String className = getCamelCaseType(operandType1)
+ "Col" + operatorName + getCamelCaseType(operandType2) + "Scalar";
String returnType = getArithmeticReturnType(operandType1, operandType2);
- generateColumnBinaryOperatorScalar(tdesc, returnType, className);
+ generateColumnArithmeticOperatorScalar(tdesc, returnType, className);
}
private void generateColumnArithmeticScalarDecimal(String[] tdesc) throws IOException {
@@ -1373,7 +1793,7 @@ public class GenVectorCode extends Task
String className = getCamelCaseType(operandType1)
+ "Scalar" + operatorName + getCamelCaseType(operandType2) + "Column";
String returnType = getArithmeticReturnType(operandType1, operandType2);
- generateScalarBinaryOperatorColumn(tdesc, returnType, className);
+ generateScalarArithmeticOperatorColumn(tdesc, returnType, className);
}
private void generateFilterDecimalColumnCompareScalar(String[] tdesc) throws IOException {
Modified: hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java?rev=1622763&r1=1622762&r2=1622763&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java (original)
+++ hive/branches/tez/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/HiveEndPoint.java Fri Sep 5 19:15:44 2014
@@ -343,7 +343,10 @@ public class HiveEndPoint {
if (ep.partitionVals.isEmpty()) {
return;
}
- SessionState state = SessionState.start(new CliSessionState(conf));
+ SessionState localSession = null;
+ if(SessionState.get() == null) {
+ localSession = SessionState.start(new CliSessionState(conf));
+ }
Driver driver = new Driver(conf);
try {
@@ -372,7 +375,9 @@ public class HiveEndPoint {
} finally {
driver.close();
try {
- state.close();
+ if(localSession != null) {
+ localSession.close();
+ }
} catch (IOException e) {
LOG.warn("Error closing SessionState used to run Hive DDL.");
}
@@ -563,11 +568,14 @@ public class HiveEndPoint {
/**
* Get Id of currently open transaction
- * @return
+ * @return -1 if there is no open TX
*/
@Override
public Long getCurrentTxnId() {
- return txnIds.get(currentTxnIndex);
+ if(currentTxnIndex >= 0) {
+ return txnIds.get(currentTxnIndex);
+ }
+ return -1L;
}
/**
Modified: hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java?rev=1622763&r1=1622762&r2=1622763&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java (original)
+++ hive/branches/tez/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java Fri Sep 5 19:15:44 2014
@@ -54,6 +54,8 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
@@ -67,6 +69,7 @@ import java.util.Map;
public class TestStreaming {
+ private static final Logger LOG = LoggerFactory.getLogger(TestStreaming.class);
public static class RawFileSystem extends RawLocalFileSystem {
private static final URI NAME;
@@ -636,18 +639,25 @@ public class TestStreaming {
connection.close();
}
- class WriterThd extends Thread {
+ private static class WriterThd extends Thread {
- private StreamingConnection conn;
- private HiveEndPoint ep;
- private DelimitedInputWriter writer;
- private String data;
+ private final StreamingConnection conn;
+ private final DelimitedInputWriter writer;
+ private final String data;
+ private Throwable error;
WriterThd(HiveEndPoint ep, String data) throws Exception {
- this.ep = ep;
+ super("Writer_" + data);
writer = new DelimitedInputWriter(fieldNames, ",", ep);
conn = ep.newConnection(false);
this.data = data;
+ setUncaughtExceptionHandler(new UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread thread, Throwable throwable) {
+ error = throwable;
+ LOG.error("Thread " + thread.getName() + " died: " + throwable.getMessage(), throwable);
+ }
+ });
}
@Override
@@ -668,14 +678,14 @@ public class TestStreaming {
try {
txnBatch.close();
} catch (Exception e) {
+ LOG.error("txnBatch.close() failed: " + e.getMessage(), e);
conn.close();
- throw new RuntimeException(e);
}
}
try {
conn.close();
} catch (Exception e) {
- throw new RuntimeException(e);
+ LOG.error("conn.close() failed: " + e.getMessage(), e);
}
}
@@ -685,18 +695,23 @@ public class TestStreaming {
@Test
public void testConcurrentTransactionBatchCommits() throws Exception {
final HiveEndPoint ep = new HiveEndPoint(metaStoreURI, dbName, tblName, partitionVals);
- WriterThd t1 = new WriterThd(ep, "1,Matrix");
- WriterThd t2 = new WriterThd(ep, "2,Gandhi");
- WriterThd t3 = new WriterThd(ep, "3,Silence");
-
- t1.start();
- t2.start();
- t3.start();
-
- t1.join();
- t2.join();
- t3.join();
-
+ List<WriterThd> writers = new ArrayList<WriterThd>(3);
+ writers.add(new WriterThd(ep, "1,Matrix"));
+ writers.add(new WriterThd(ep, "2,Gandhi"));
+ writers.add(new WriterThd(ep, "3,Silence"));
+
+ for(WriterThd w : writers) {
+ w.start();
+ }
+ for(WriterThd w : writers) {
+ w.join();
+ }
+ for(WriterThd w : writers) {
+ if(w.error != null) {
+ Assert.assertFalse("Writer thread" + w.getName() + " died: " + w.error.getMessage() +
+ " See log file for stack trace", true);
+ }
+ }
}
// delete db and all tables in it
Modified: hive/branches/tez/itests/hive-unit/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/pom.xml?rev=1622763&r1=1622762&r2=1622763&view=diff
==============================================================================
--- hive/branches/tez/itests/hive-unit/pom.xml (original)
+++ hive/branches/tez/itests/hive-unit/pom.xml Fri Sep 5 19:15:44 2014
@@ -53,6 +53,16 @@
<artifactId>hive-exec</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive.hcatalog</groupId>
+ <artifactId>hive-hcatalog-streaming</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- dependencies are always listed in sorted order by groupId, artifectId -->
<!-- test intra-project -->
Modified: hive/branches/tez/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/if/hive_metastore.thrift?rev=1622763&r1=1622762&r2=1622763&view=diff
==============================================================================
--- hive/branches/tez/metastore/if/hive_metastore.thrift (original)
+++ hive/branches/tez/metastore/if/hive_metastore.thrift Fri Sep 5 19:15:44 2014
@@ -273,6 +273,32 @@ struct Partition {
8: optional PrincipalPrivilegeSet privileges
}
+struct PartitionWithoutSD {
+ 1: list<string> values // string value is converted to appropriate partition key type
+ 2: i32 createTime,
+ 3: i32 lastAccessTime,
+ 4: string relativePath,
+ 5: map<string, string> parameters,
+ 6: optional PrincipalPrivilegeSet privileges
+}
+
+struct PartitionSpecWithSharedSD {
+ 1: list<PartitionWithoutSD> partitions,
+ 2: StorageDescriptor sd,
+}
+
+struct PartitionListComposingSpec {
+ 1: list<Partition> partitions
+}
+
+struct PartitionSpec {
+ 1: string dbName,
+ 2: string tableName,
+ 3: string rootPath,
+ 4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec,
+ 5: optional PartitionListComposingSpec partitionList
+}
+
struct Index {
1: string indexName, // unique with in the whole database namespace
2: string indexHandlerClass, // reserved
@@ -793,6 +819,8 @@ service ThriftHiveMetastore extends fb30
3:MetaException o3)
i32 add_partitions(1:list<Partition> new_parts)
throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
+ i32 add_partitions_pspec(1:list<PartitionSpec> new_parts)
+ throws(1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
Partition append_partition(1:string db_name, 2:string tbl_name, 3:list<string> part_vals)
throws (1:InvalidObjectException o1, 2:AlreadyExistsException o2, 3:MetaException o3)
AddPartitionsResult add_partitions_req(1:AddPartitionsRequest request)
@@ -838,6 +866,9 @@ service ThriftHiveMetastore extends fb30
list<Partition> get_partitions_with_auth(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1,
4: string user_name, 5: list<string> group_names) throws(1:NoSuchObjectException o1, 2:MetaException o2)
+ list<PartitionSpec> get_partitions_pspec(1:string db_name, 2:string tbl_name, 3:i32 max_parts=-1)
+ throws(1:NoSuchObjectException o1, 2:MetaException o2)
+
list<string> get_partition_names(1:string db_name, 2:string tbl_name, 3:i16 max_parts=-1)
throws(1:MetaException o2)
@@ -862,6 +893,11 @@ service ThriftHiveMetastore extends fb30
3:string filter, 4:i16 max_parts=-1)
throws(1:MetaException o1, 2:NoSuchObjectException o2)
+ // List partitions as PartitionSpec instances.
+ list<PartitionSpec> get_part_specs_by_filter(1:string db_name 2:string tbl_name
+ 3:string filter, 4:i32 max_parts=-1)
+ throws(1:MetaException o1, 2:NoSuchObjectException o2)
+
// get the partitions matching the given partition filter
// unlike get_partitions_by_filter, takes serialized hive expression, and with that can work
// with any filter (get_partitions_by_filter only works if the filter can be pushed down to JDOQL.