You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/02/25 08:59:20 UTC
svn commit: r1571600 [15/46] - in /hive/branches/tez: ./
ant/src/org/apache/hadoop/hive/ant/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/hadoop/hive/conf/ common/src/java...
Modified: hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt (original)
+++ hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFAvg.txt Tue Feb 25 07:58:52 2014
@@ -54,7 +54,11 @@ public class <ClassName> extends VectorA
transient private double sum;
transient private long count;
- transient private boolean isNull;
+
+ /**
+ * Value is explicitly (re)initialized in reset()
+ */
+ transient private boolean isNull = true;
public void sumValue(<ValueType> value) {
if (isNull) {
Modified: hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt (original)
+++ hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMax.txt Tue Feb 25 07:58:52 2014
@@ -49,7 +49,11 @@ public class <ClassName> extends VectorA
private static final long serialVersionUID = 1L;
transient private <ValueType> value;
- transient private boolean isNull;
+
+ /**
+ * Value is explicitly (re)initialized in reset()
+ */
+ transient private boolean isNull = true;
public void checkValue(<ValueType> value) {
if (isNull) {
Modified: hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt (original)
+++ hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxString.txt Tue Feb 25 07:58:52 2014
@@ -35,15 +35,15 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.io.Text;
/**
-* <ClassName>. Vectorized implementation for MIN/MAX aggregates.
+* <ClassName>. Vectorized implementation for MIN/MAX aggregates.
*/
-@Description(name = "<DescriptionName>",
+@Description(name = "<DescriptionName>",
value = "<DescriptionValue>")
public class <ClassName> extends VectorAggregateExpression {
private static final long serialVersionUID = 1L;
- /**
+ /**
* class for storing the current aggregate value.
*/
static private final class Aggregation implements AggregationBuffer {
@@ -53,7 +53,11 @@ public class <ClassName> extends VectorA
transient private final static int MIN_BUFFER_SIZE = 16;
transient private byte[] bytes = new byte[MIN_BUFFER_SIZE];
transient private int length;
- transient private boolean isNull;
+
+ /**
+ * Value is explicitly (re)initialized in reset()
+ */
+ transient private boolean isNull = true;
public void checkValue(byte[] bytes, int start, int length) {
if (isNull) {
@@ -65,7 +69,7 @@ public class <ClassName> extends VectorA
assign(bytes, start, length);
}
}
-
+
public void assign(byte[] bytes, int start, int length) {
// Avoid new allocation if possible
if (this.bytes.length < length) {
@@ -80,10 +84,10 @@ public class <ClassName> extends VectorA
return model.lengthForByteArrayOfSize(bytes.length);
}
}
-
+
private VectorExpression inputExpression;
transient private Text result;
-
+
public <ClassName>(VectorExpression inputExpression) {
this();
this.inputExpression = inputExpression;
@@ -93,7 +97,7 @@ public class <ClassName> extends VectorA
super();
result = new Text();
}
-
+
private Aggregation getCurrentAggregationBuffer(
VectorAggregationBufferRow[] aggregationBufferSets,
int aggregrateIndex,
@@ -102,21 +106,21 @@ public class <ClassName> extends VectorA
Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregrateIndex);
return myagg;
}
-
+
@Override
public void aggregateInputSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
- int aggregrateIndex,
+ int aggregrateIndex,
VectorizedRowBatch batch) throws HiveException {
-
+
int batchSize = batch.size;
-
+
if (batchSize == 0) {
return;
}
-
+
inputExpression.evaluate(batch);
-
+
BytesColumnVector inputColumn = (BytesColumnVector)batch.
cols[this.inputExpression.getOutputColumn()];
@@ -164,12 +168,12 @@ public class <ClassName> extends VectorA
int length = inputColumn.length[0];
for (int i=0; i < batchSize; ++i) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregrateIndex,
i);
myagg.checkValue(bytes, start, length);
}
- }
+ }
private void iterateNoNullsSelectionWithAggregationSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
@@ -177,11 +181,11 @@ public class <ClassName> extends VectorA
BytesColumnVector inputColumn,
int[] selection,
int batchSize) {
-
+
for (int i=0; i < batchSize; ++i) {
int row = selection[i];
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregrateIndex,
i);
myagg.checkValue(inputColumn.vector[row],
@@ -197,7 +201,7 @@ public class <ClassName> extends VectorA
int batchSize) {
for (int i=0; i < batchSize; ++i) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregrateIndex,
i);
myagg.checkValue(inputColumn.vector[i],
@@ -217,7 +221,7 @@ public class <ClassName> extends VectorA
int row = selection[i];
if (!inputColumn.isNull[row]) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregrateIndex,
i);
myagg.checkValue(inputColumn.vector[row],
@@ -236,7 +240,7 @@ public class <ClassName> extends VectorA
for (int i=0; i < batchSize; ++i) {
if (!inputColumn.isNull[i]) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregrateIndex,
i);
myagg.checkValue(inputColumn.vector[i],
@@ -245,24 +249,24 @@ public class <ClassName> extends VectorA
}
}
}
-
+
@Override
- public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch)
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch)
throws HiveException {
-
+
inputExpression.evaluate(batch);
-
+
BytesColumnVector inputColumn = (BytesColumnVector)batch.
cols[this.inputExpression.getOutputColumn()];
-
+
int batchSize = batch.size;
-
+
if (batchSize == 0) {
return;
}
-
+
Aggregation myagg = (Aggregation)agg;
-
+
if (inputColumn.isRepeating) {
if (inputColumn.noNulls) {
myagg.checkValue(inputColumn.vector[0],
@@ -271,7 +275,7 @@ public class <ClassName> extends VectorA
}
return;
}
-
+
if (!batch.selectedInUse && inputColumn.noNulls) {
iterateNoSelectionNoNulls(myagg, inputColumn, batchSize);
}
@@ -285,13 +289,13 @@ public class <ClassName> extends VectorA
iterateSelectionHasNulls(myagg, inputColumn, batchSize, batch.selected);
}
}
-
+
private void iterateSelectionHasNulls(
- Aggregation myagg,
- BytesColumnVector inputColumn,
+ Aggregation myagg,
+ BytesColumnVector inputColumn,
int batchSize,
int[] selected) {
-
+
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!inputColumn.isNull[i]) {
@@ -303,11 +307,11 @@ public class <ClassName> extends VectorA
}
private void iterateSelectionNoNulls(
- Aggregation myagg,
- BytesColumnVector inputColumn,
- int batchSize,
+ Aggregation myagg,
+ BytesColumnVector inputColumn,
+ int batchSize,
int[] selected) {
-
+
for (int i=0; i< batchSize; ++i) {
myagg.checkValue(inputColumn.vector[i],
inputColumn.start[i],
@@ -316,10 +320,10 @@ public class <ClassName> extends VectorA
}
private void iterateNoSelectionHasNulls(
- Aggregation myagg,
- BytesColumnVector inputColumn,
+ Aggregation myagg,
+ BytesColumnVector inputColumn,
int batchSize) {
-
+
for (int i=0; i< batchSize; ++i) {
if (!inputColumn.isNull[i]) {
myagg.checkValue(inputColumn.vector[i],
@@ -330,7 +334,7 @@ public class <ClassName> extends VectorA
}
private void iterateNoSelectionNoNulls(
- Aggregation myagg,
+ Aggregation myagg,
BytesColumnVector inputColumn,
int batchSize) {
for (int i=0; i< batchSize; ++i) {
@@ -363,7 +367,7 @@ public class <ClassName> extends VectorA
return result;
}
}
-
+
@Override
public ObjectInspector getOutputObjectInspector() {
return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
@@ -378,7 +382,7 @@ public class <ClassName> extends VectorA
model.primitive1()*2,
model.memoryAlign());
}
-
+
@Override
public boolean hasVariableSize() {
return true;
Modified: hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt (original)
+++ hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFSum.txt Tue Feb 25 07:58:52 2014
@@ -50,7 +50,11 @@ public class <ClassName> extends VectorA
private static final long serialVersionUID = 1L;
transient private <ValueType> sum;
- transient private boolean isNull;
+
+ /**
+ * Value is explicitly (re)initialized in reset()
+ */
+ transient private boolean isNull = true;
public void sumValue(<ValueType> value) {
if (isNull) {
Modified: hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt (original)
+++ hive/branches/tez/ql/src/gen/vectorization/UDAFTemplates/VectorUDAFVar.txt Tue Feb 25 07:58:52 2014
@@ -38,16 +38,16 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
/**
-* <ClassName>. Vectorized implementation for VARIANCE aggregates.
+* <ClassName>. Vectorized implementation for VARIANCE aggregates.
*/
@Description(name = "<DescriptionName>",
value = "<DescriptionValue>")
public class <ClassName> extends VectorAggregateExpression {
private static final long serialVersionUID = 1L;
-
- /**
- /* class for storing the current aggregate value.
+
+ /**
+ /* class for storing the current aggregate value.
*/
private static final class Aggregation implements AggregationBuffer {
@@ -56,8 +56,12 @@ public class <ClassName> extends VectorA
transient private double sum;
transient private long count;
transient private double variance;
- transient private boolean isNull;
-
+
+ /**
+ * Value is explicitly (re)initialized in reset() (despite the init() bellow...)
+ */
+ transient private boolean isNull = true;
+
public void init() {
isNull = false;
sum = 0;
@@ -70,16 +74,16 @@ public class <ClassName> extends VectorA
throw new UnsupportedOperationException();
}
}
-
+
private VectorExpression inputExpression;
transient private LongWritable resultCount;
transient private DoubleWritable resultSum;
transient private DoubleWritable resultVariance;
transient private Object[] partialResult;
-
+
transient private ObjectInspector soi;
-
-
+
+
public <ClassName>(VectorExpression inputExpression) {
this();
this.inputExpression = inputExpression;
@@ -120,32 +124,32 @@ public class <ClassName> extends VectorA
return myagg;
}
-
+
@Override
public void aggregateInputSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
- int aggregateIndex,
+ int aggregateIndex,
VectorizedRowBatch batch) throws HiveException {
-
+
inputExpression.evaluate(batch);
-
+
<InputColumnVectorType> inputVector = (<InputColumnVectorType>)batch.
cols[this.inputExpression.getOutputColumn()];
-
+
int batchSize = batch.size;
-
+
if (batchSize == 0) {
return;
}
-
+
<ValueType>[] vector = inputVector.vector;
-
+
if (inputVector.isRepeating) {
if (inputVector.noNulls || !inputVector.isNull[0]) {
iterateRepeatingNoNullsWithAggregationSelection(
aggregationBufferSets, aggregateIndex, vector[0], batchSize);
}
- }
+ }
else if (!batch.selectedInUse && inputVector.noNulls) {
iterateNoSelectionNoNullsWithAggregationSelection(
aggregationBufferSets, aggregateIndex, vector, batchSize);
@@ -160,46 +164,46 @@ public class <ClassName> extends VectorA
}
else {
iterateSelectionHasNullsWithAggregationSelection(
- aggregationBufferSets, aggregateIndex, vector, batchSize,
+ aggregationBufferSets, aggregateIndex, vector, batchSize,
inputVector.isNull, batch.selected);
}
-
+
}
-
+
private void iterateRepeatingNoNullsWithAggregationSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
- int aggregateIndex,
- double value,
+ int aggregateIndex,
+ double value,
int batchSize) {
for (int i=0; i<batchSize; ++i) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregateIndex,
i);
if (myagg.isNull) {
myagg.init ();
}
myagg.sum += value;
- myagg.count += 1;
+ myagg.count += 1;
if(myagg.count > 1) {
double t = myagg.count*value - myagg.sum;
myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
}
}
}
-
+
private void iterateSelectionHasNullsWithAggregationSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
- int aggregateIndex,
- <ValueType>[] vector,
+ int aggregateIndex,
+ <ValueType>[] vector,
int batchSize,
- boolean[] isNull,
+ boolean[] isNull,
int[] selected) {
-
+
for (int j=0; j< batchSize; ++j) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregateIndex,
j);
int i = selected[j];
@@ -220,14 +224,14 @@ public class <ClassName> extends VectorA
private void iterateSelectionNoNullsWithAggregationSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
- int aggregateIndex,
- <ValueType>[] vector,
- int batchSize,
+ int aggregateIndex,
+ <ValueType>[] vector,
+ int batchSize,
int[] selected) {
for (int i=0; i< batchSize; ++i) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregateIndex,
i);
double value = vector[selected[i]];
@@ -245,20 +249,20 @@ public class <ClassName> extends VectorA
private void iterateNoSelectionHasNullsWithAggregationSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
- int aggregateIndex,
- <ValueType>[] vector,
+ int aggregateIndex,
+ <ValueType>[] vector,
int batchSize,
boolean[] isNull) {
-
+
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregateIndex,
i);
double value = vector[i];
if (myagg.isNull) {
- myagg.init ();
+ myagg.init ();
}
myagg.sum += value;
myagg.count += 1;
@@ -272,13 +276,13 @@ public class <ClassName> extends VectorA
private void iterateNoSelectionNoNullsWithAggregationSelection(
VectorAggregationBufferRow[] aggregationBufferSets,
- int aggregateIndex,
- <ValueType>[] vector,
+ int aggregateIndex,
+ <ValueType>[] vector,
int batchSize) {
for (int i=0; i<batchSize; ++i) {
Aggregation myagg = getCurrentAggregationBuffer(
- aggregationBufferSets,
+ aggregationBufferSets,
aggregateIndex,
i);
if (myagg.isNull) {
@@ -295,29 +299,29 @@ public class <ClassName> extends VectorA
}
@Override
- public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch)
+ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch)
throws HiveException {
-
+
inputExpression.evaluate(batch);
-
+
<InputColumnVectorType> inputVector = (<InputColumnVectorType>)batch.
cols[this.inputExpression.getOutputColumn()];
-
+
int batchSize = batch.size;
-
+
if (batchSize == 0) {
return;
}
-
+
Aggregation myagg = (Aggregation)agg;
<ValueType>[] vector = inputVector.vector;
-
+
if (inputVector.isRepeating) {
if (inputVector.noNulls) {
iterateRepeatingNoNulls(myagg, vector[0], batchSize);
}
- }
+ }
else if (!batch.selectedInUse && inputVector.noNulls) {
iterateNoSelectionNoNulls(myagg, vector, batchSize);
}
@@ -333,40 +337,40 @@ public class <ClassName> extends VectorA
}
private void iterateRepeatingNoNulls(
- Aggregation myagg,
- double value,
+ Aggregation myagg,
+ double value,
int batchSize) {
-
+
if (myagg.isNull) {
myagg.init ();
}
-
+
// TODO: conjure a formula w/o iterating
//
-
+
myagg.sum += value;
- myagg.count += 1;
+ myagg.count += 1;
if(myagg.count > 1) {
double t = myagg.count*value - myagg.sum;
myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
}
-
+
// We pulled out i=0 so we can remove the count > 1 check in the loop
for (int i=1; i<batchSize; ++i) {
myagg.sum += value;
- myagg.count += 1;
+ myagg.count += 1;
double t = myagg.count*value - myagg.sum;
myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
}
}
-
+
private void iterateSelectionHasNulls(
- Aggregation myagg,
- <ValueType>[] vector,
+ Aggregation myagg,
+ <ValueType>[] vector,
int batchSize,
- boolean[] isNull,
+ boolean[] isNull,
int[] selected) {
-
+
for (int j=0; j< batchSize; ++j) {
int i = selected[j];
if (!isNull[i]) {
@@ -385,11 +389,11 @@ public class <ClassName> extends VectorA
}
private void iterateSelectionNoNulls(
- Aggregation myagg,
- <ValueType>[] vector,
- int batchSize,
+ Aggregation myagg,
+ <ValueType>[] vector,
+ int batchSize,
int[] selected) {
-
+
if (myagg.isNull) {
myagg.init ();
}
@@ -401,7 +405,7 @@ public class <ClassName> extends VectorA
double t = myagg.count*value - myagg.sum;
myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
}
-
+
// i=0 was pulled out to remove the count > 1 check in the loop
//
for (int i=1; i< batchSize; ++i) {
@@ -414,16 +418,16 @@ public class <ClassName> extends VectorA
}
private void iterateNoSelectionHasNulls(
- Aggregation myagg,
- <ValueType>[] vector,
+ Aggregation myagg,
+ <ValueType>[] vector,
int batchSize,
boolean[] isNull) {
-
+
for(int i=0;i<batchSize;++i) {
if (!isNull[i]) {
double value = vector[i];
if (myagg.isNull) {
- myagg.init ();
+ myagg.init ();
}
myagg.sum += value;
myagg.count += 1;
@@ -436,10 +440,10 @@ public class <ClassName> extends VectorA
}
private void iterateNoSelectionNoNulls(
- Aggregation myagg,
- <ValueType>[] vector,
+ Aggregation myagg,
+ <ValueType>[] vector,
int batchSize) {
-
+
if (myagg.isNull) {
myagg.init ();
}
@@ -447,12 +451,12 @@ public class <ClassName> extends VectorA
double value = vector[0];
myagg.sum += value;
myagg.count += 1;
-
+
if(myagg.count > 1) {
double t = myagg.count*value - myagg.sum;
myagg.variance += (t*t) / ((double)myagg.count*(myagg.count-1));
}
-
+
// i=0 was pulled out to remove count > 1 check
for (int i=1; i<batchSize; ++i) {
value = vector[i];
@@ -515,6 +519,6 @@ public class <ClassName> extends VectorA
public void setInputExpression(VectorExpression inputExpression) {
this.inputExpression = inputExpression;
- }
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Tue Feb 25 07:58:52 2014
@@ -55,7 +55,6 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
import org.apache.hadoop.hive.ql.hooks.Entity;
-import org.apache.hadoop.hive.ql.hooks.Entity.Type;
import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.Hook;
import org.apache.hadoop.hive.ql.hooks.HookContext;
@@ -594,7 +593,7 @@ public class Driver implements CommandPr
continue;
}
Table tbl = read.getTable();
- if ((read.getPartition() != null) || (tbl.isPartitioned())) {
+ if ((read.getPartition() != null) || (tbl != null && tbl.isPartitioned())) {
String tblName = tbl.getTableName();
if (tableUsePartLevelAuth.get(tblName) == null) {
boolean usePartLevelPriv = (tbl.getParameters().get(
@@ -728,24 +727,47 @@ public class Driver implements CommandPr
HivePrivilegeObjectType privObjType =
AuthorizationUtils.getHivePrivilegeObjectType(privObject.getType());
- //support for authorization on partitions or uri needs to be added
- HivePrivilegeObject hPrivObject = new HivePrivilegeObject(privObjType,
- getDataBaseName(privObject),
- privObject.getTable() == null ? null : privObject.getTable().getTableName());
+ if(privObject instanceof ReadEntity && !((ReadEntity)privObject).isDirect()){
+ // In case of views, the underlying views or tables are not direct dependencies
+ // and are not used for authorization checks.
+ // This ReadEntity represents one of the underlying tables/views, so skip it.
+ // See description of the isDirect in ReadEntity
+ continue;
+ }
+ if(privObject instanceof WriteEntity && ((WriteEntity)privObject).isTempURI()){
+ //do not authorize temporary uris
+ continue;
+ }
+
+ //support for authorization on partitions needs to be added
+ String dbname = null;
+ String tableURI = null;
+ switch(privObject.getType()){
+ case DATABASE:
+ dbname = privObject.getDatabase() == null ? null : privObject.getDatabase().getName();
+ break;
+ case TABLE:
+ dbname = privObject.getTable() == null ? null : privObject.getTable().getDbName();
+ tableURI = privObject.getTable() == null ? null : privObject.getTable().getTableName();
+ break;
+ case DFS_DIR:
+ case LOCAL_DIR:
+ tableURI = privObject.getD();
+ break;
+ case DUMMYPARTITION:
+ case PARTITION:
+ // not currently handled
+ break;
+ default:
+ throw new AssertionError("Unexpected object type");
+ }
+
+ HivePrivilegeObject hPrivObject = new HivePrivilegeObject(privObjType, dbname, tableURI);
hivePrivobjs.add(hPrivObject);
}
return hivePrivobjs;
}
-
- private String getDataBaseName(Entity privObject) {
- if(privObject.getType() == Type.DATABASE){
- return privObject.getDatabase() == null ? null : privObject.getDatabase().getName();
- } else {
- return privObject.getTable() == null ? null : privObject.getTable().getDbName();
- }
- }
-
private HiveOperationType getHiveOperationType(HiveOperation op) {
return HiveOperationType.valueOf(op.name());
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java Tue Feb 25 07:58:52 2014
@@ -25,6 +25,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.antlr.runtime.tree.Tree;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ASTNodeOrigin;
@@ -374,6 +375,7 @@ public enum ErrorMsg {
UNNSUPPORTED_AUTHORIZATION_PRINCIPAL_TYPE_GROUP(10254,
"Principal type GROUP is not supported in this authorization setting", "28000"),
INVALID_TABLE_NAME(10255, "Invalid table name {0}", true),
+ INSERT_INTO_IMMUTABLE_TABLE(10256, "Inserting into a non-empty immutable table is not allowed"),
//========================== 20000 range starts here ========================//
SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
@@ -431,7 +433,9 @@ public enum ErrorMsg {
STATSAGGREGATOR_MISSED_SOMESTATS(30016,
"Stats type {0} is missing from stats aggregator. If you don't want the query " +
"to fail because of this, set hive.stats.atomic=false", true),
- STATS_SKIPPING_BY_ERROR(30017, "Skipping stats aggregation by error {0}", true);
+ STATS_SKIPPING_BY_ERROR(30017, "Skipping stats aggregation by error {0}", true),
+ ORC_CORRUPTED_READ(30018, "Corruption in ORC data encountered. To skip reading corrupted "
+ + "data, set " + HiveConf.ConfVars.HIVE_ORC_SKIP_CORRUPT_DATA + " to true");
;
private int errorCode;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Tue Feb 25 07:58:52 2014
@@ -176,7 +176,6 @@ import org.apache.hadoop.hive.shims.Shim
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.thrift.TException;
import org.stringtemplate.v4.ST;
/**
@@ -658,7 +657,7 @@ public class DDLTask extends Task<DDLWor
//only grantInfo is used
HiveObjectPrivilege thriftObjectPriv = new HiveObjectPrivilege(new HiveObjectRef(
AuthorizationUtils.getThriftHiveObjType(privObj.getType()),privObj.getDbname(),
- privObj.getTableviewname(),null,null), principal.getName(),
+ privObj.getTableViewURI(),null,null), principal.getName(),
AuthorizationUtils.getThriftPrincipalType(principal.getType()), grantInfo);
privList.add(thriftObjectPriv);
}
@@ -873,9 +872,7 @@ public class DDLTask extends Task<DDLWor
}
private HivePrivilegeObjectType getPrivObjectType(PrivilegeObjectDesc privSubjectDesc) {
- //TODO: This needs to change to support view once view grant/revoke is supported as
- // part of HIVE-6181
- return privSubjectDesc.getTable() ? HivePrivilegeObjectType.TABLE : HivePrivilegeObjectType.DATABASE;
+ return privSubjectDesc.getTable() ? HivePrivilegeObjectType.TABLE_OR_VIEW : HivePrivilegeObjectType.DATABASE;
}
private List<HivePrivilege> getHivePrivileges(List<PrivilegeDesc> privileges) {
@@ -971,7 +968,7 @@ public class DDLTask extends Task<DDLWor
break;
case SHOW_ROLES:
List<String> allRoles = authorizer.getAllRoles();
- writeListToFile(allRoles, roleDDLDesc.getResFile());
+ writeListToFileAfterSort(allRoles, roleDDLDesc.getResFile());
break;
case SHOW_CURRENT_ROLE:
List<HiveRole> currentRoles = authorizer.getCurrentRoles();
@@ -979,7 +976,7 @@ public class DDLTask extends Task<DDLWor
for (HiveRole role : currentRoles) {
roleNames.add(role.getRoleName());
}
- writeListToFile(roleNames, roleDDLDesc.getResFile());
+ writeListToFileAfterSort(roleNames, roleDDLDesc.getResFile());
break;
case SET_ROLE:
authorizer.setCurrentRole(roleDDLDesc.getName());
@@ -998,7 +995,8 @@ public class DDLTask extends Task<DDLWor
* @param resFile
* @throws IOException
*/
- private void writeListToFile(List<String> entries, String resFile) throws IOException {
+ private void writeListToFileAfterSort(List<String> entries, String resFile) throws IOException {
+ Collections.sort(entries);
StringBuilder sb = new StringBuilder();
for(String entry : entries){
sb.append(entry);
@@ -2836,12 +2834,10 @@ public class DDLTask extends Task<DDLWor
if(descDatabase.isExt()) {
params = database.getParameters();
}
-
- formatter.showDatabaseDescription(outStream,
- database.getName(),
- database.getDescription(),
- database.getLocationUri(),
- params);
+ PrincipalType ownerType = database.getOwnerType();
+ formatter.showDatabaseDescription(outStream, database.getName(),
+ database.getDescription(), database.getLocationUri(),
+ database.getOwnerName(), (null == ownerType) ? null : ownerType.name(), params);
}
outStream.close();
outStream = null;
@@ -3632,12 +3628,7 @@ public class DDLTask extends Task<DDLWor
*/
private boolean updateModifiedParameters(Map<String, String> params, HiveConf conf) throws HiveException {
String user = null;
- try {
- user = conf.getUser();
- } catch (IOException e) {
- throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "Unable to get current user");
- }
-
+ user = SessionState.getUserFromAuthenticator();
params.put("last_modified_by", user);
params.put("last_modified_time", Long.toString(System.currentTimeMillis() / 1000));
return true;
@@ -3673,6 +3664,8 @@ public class DDLTask extends Task<DDLWor
database.setDescription(crtDb.getComment());
database.setLocationUri(crtDb.getLocationUri());
database.setParameters(crtDb.getDatabaseProperties());
+ database.setOwnerName(SessionState.getUserFromAuthenticator());
+ database.setOwnerType(PrincipalType.USER);
try {
db.createDatabase(database, crtDb.getIfNotExists());
}
@@ -4137,11 +4130,7 @@ public class DDLTask extends Task<DDLWor
}
private int setGenericTableAttributes(Table tbl) throws HiveException {
- try {
- tbl.setOwner(conf.getUser());
- } catch (IOException e) {
- throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "Unable to get current user");
- }
+ tbl.setOwner(SessionState.getUserFromAuthenticator());
// set create time
tbl.setCreateTime((int) (System.currentTimeMillis() / 1000));
return 0;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java Tue Feb 25 07:58:52 2014
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.exec;
import java.io.IOException;
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Tue Feb 25 07:58:52 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec;
+import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Method;
import java.net.URL;
@@ -39,7 +40,12 @@ import javax.xml.parsers.DocumentBuilder
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.ql.exec.FunctionUtils.UDFClassType;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -519,9 +525,62 @@ public final class FunctionRegistry {
}
}
+ private static FunctionInfo getFunctionInfoFromMetastore(String functionName) {
+ FunctionInfo ret = null;
+
+ try {
+ String dbName;
+ String fName;
+ if (FunctionUtils.isQualifiedFunctionName(functionName)) {
+ String[] parts = FunctionUtils.splitQualifiedFunctionName(functionName);
+ dbName = parts[0];
+ fName = parts[1];
+ } else {
+ // otherwise, qualify using current db
+ dbName = SessionState.get().getCurrentDatabase();
+ fName = functionName;
+ }
+
+ // Try looking up function in the metastore
+ HiveConf conf = SessionState.get().getConf();
+ Function func = Hive.get(conf).getFunction(dbName, fName);
+ if (func != null) {
+ // Found UDF in metastore - now add it to the function registry
+ // At this point we should add any relevant jars that would be needed for the UDf.
+ try {
+ FunctionTask.addFunctionResources(func.getResourceUris());
+ } catch (Exception e) {
+ LOG.error("Unable to load resources for " + dbName + "." + fName + ":" + e);
+ return null;
+ }
+
+ Class<?> udfClass = Class.forName(func.getClassName(), true, JavaUtils.getClassLoader());
+ if (registerTemporaryFunction(functionName, udfClass)) {
+ ret = mFunctions.get(functionName);
+ } else {
+ LOG.error(func.getClassName() + " is not a valid UDF class and was not registered.");
+ }
+ }
+ } catch (HiveException e) {
+ LOG.info("Unable to lookup UDF in metastore: " + e);
+ } catch (ClassNotFoundException e) {
+ // Lookup of UDf class failed
+ LOG.error("Unable to load UDF class: " + e);
+ }
+
+ return ret;
+ }
+
private static <T> T getQualifiedFunctionInfo(Map<String, T> mFunctions, String functionName) {
T functionInfo = mFunctions.get(functionName);
- // Eventually this would check metastore for registered functions.
+ if (functionInfo == null) {
+ // Try looking up in metastore.
+ FunctionInfo fi = getFunctionInfoFromMetastore(functionName);
+ if (fi != null) {
+ // metastore lookup resulted in function getting added to mFunctions, try again
+ functionInfo = mFunctions.get(functionName);
+ }
+ }
return functionInfo;
}
@@ -554,7 +613,33 @@ public final class FunctionRegistry {
* @return set of strings contains function names
*/
public static Set<String> getFunctionNames() {
- return mFunctions.keySet();
+ return getFunctionNames(true);
+ }
+
+ private static Set<String> getFunctionNames(boolean searchMetastore) {
+ Set<String> functionNames = mFunctions.keySet();
+ if (searchMetastore) {
+ functionNames = new HashSet<String>(functionNames);
+ try {
+ Hive db = getHive();
+ List<String> dbNames = db.getAllDatabases();
+
+ for (String dbName : dbNames) {
+ List<String> funcNames = db.getFunctions(dbName, "*");
+ for (String funcName : funcNames) {
+ functionNames.add(FunctionUtils.qualifyFunctionName(funcName, dbName));
+ }
+ }
+ } catch (Exception e) {
+ LOG.error(e);
+ // Continue on, we can still return the functions we've gotten to this point.
+ }
+ }
+ return functionNames;
+ }
+
+ public static Hive getHive() throws HiveException {
+ return Hive.get(SessionState.get().getConf());
}
/**
@@ -1593,29 +1678,38 @@ public final class FunctionRegistry {
public static boolean registerTemporaryFunction(
String functionName, Class<?> udfClass) {
- if (UDF.class.isAssignableFrom(udfClass)) {
+ UDFClassType udfClassType = FunctionUtils.getUDFClassType(udfClass);
+ switch (udfClassType) {
+ case UDF:
FunctionRegistry.registerTemporaryUDF(
functionName, (Class<? extends UDF>) udfClass, false);
- } else if (GenericUDF.class.isAssignableFrom(udfClass)) {
+ break;
+ case GENERIC_UDF:
FunctionRegistry.registerTemporaryGenericUDF(
functionName, (Class<? extends GenericUDF>) udfClass);
- } else if (GenericUDTF.class.isAssignableFrom(udfClass)) {
+ break;
+ case GENERIC_UDTF:
FunctionRegistry.registerTemporaryGenericUDTF(
functionName, (Class<? extends GenericUDTF>) udfClass);
- } else if (UDAF.class.isAssignableFrom(udfClass)) {
+ break;
+ case UDAF:
FunctionRegistry.registerTemporaryUDAF(
functionName, (Class<? extends UDAF>) udfClass);
- } else if (GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
+ break;
+ case GENERIC_UDAF_RESOLVER:
FunctionRegistry.registerTemporaryGenericUDAF(
functionName, (GenericUDAFResolver)
ReflectionUtils.newInstance(udfClass, null));
- } else if(TableFunctionResolver.class.isAssignableFrom(udfClass)) {
+ break;
+ case TABLE_FUNCTION_RESOLVER:
FunctionRegistry.registerTableFunction(
functionName, (Class<? extends TableFunctionResolver>)udfClass);
- } else {
+ break;
+ default:
return false;
}
return true;
+
}
/**
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java Tue Feb 25 07:58:52 2014
@@ -18,13 +18,27 @@
package org.apache.hadoop.hive.ql.exec;
+import static org.apache.hadoop.util.StringUtils.stringifyException;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.metastore.api.ResourceType;
+import org.apache.hadoop.hive.metastore.api.ResourceUri;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.FunctionUtils.FunctionType;
+import org.apache.hadoop.hive.ql.exec.FunctionUtils.UDFClassType;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.CreateFunctionDesc;
import org.apache.hadoop.hive.ql.plan.DropFunctionDesc;
@@ -32,6 +46,7 @@ import org.apache.hadoop.hive.ql.plan.Cr
import org.apache.hadoop.hive.ql.plan.DropMacroDesc;
import org.apache.hadoop.hive.ql.plan.FunctionWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
@@ -62,12 +77,32 @@ public class FunctionTask extends Task<F
public int execute(DriverContext driverContext) {
CreateFunctionDesc createFunctionDesc = work.getCreateFunctionDesc();
if (createFunctionDesc != null) {
- return createFunction(createFunctionDesc);
+ if (createFunctionDesc.isTemp()) {
+ return createTemporaryFunction(createFunctionDesc);
+ } else {
+ try {
+ return createPermanentFunction(Hive.get(conf), createFunctionDesc);
+ } catch (Exception e) {
+ setException(e);
+ LOG.error(stringifyException(e));
+ return 1;
+ }
+ }
}
DropFunctionDesc dropFunctionDesc = work.getDropFunctionDesc();
if (dropFunctionDesc != null) {
- return dropFunction(dropFunctionDesc);
+ if (dropFunctionDesc.isTemp()) {
+ return dropTemporaryFunction(dropFunctionDesc);
+ } else {
+ try {
+ return dropPermanentFunction(Hive.get(conf), dropFunctionDesc);
+ } catch (Exception e) {
+ setException(e);
+ LOG.error(stringifyException(e));
+ return 1;
+ }
+ }
}
CreateMacroDesc createMacroDesc = work.getCreateMacroDesc();
@@ -82,8 +117,67 @@ public class FunctionTask extends Task<F
return 0;
}
- private int createFunction(CreateFunctionDesc createFunctionDesc) {
+ private int createPermanentFunction(Hive db, CreateFunctionDesc createFunctionDesc)
+ throws HiveException, IOException {
+ String[] qualifiedNameParts = FunctionUtils.getQualifiedFunctionNameParts(
+ createFunctionDesc.getFunctionName());
+ String dbName = qualifiedNameParts[0];
+ String funcName = qualifiedNameParts[1];
+ String registeredName = FunctionUtils.qualifyFunctionName(funcName, dbName);
+ String className = createFunctionDesc.getClassName();
+ boolean addedToRegistry = false;
+
+ try {
+ // For permanent functions, check for any resources from local filesystem.
+ checkLocalFunctionResources(db, createFunctionDesc.getResources());
+
+ // Add any required resources
+ addFunctionResources(createFunctionDesc.getResources());
+
+ // UDF class should exist
+ Class<?> udfClass = getUdfClass(createFunctionDesc);
+ if (FunctionUtils.getUDFClassType(udfClass) == UDFClassType.UNKNOWN) {
+ console.printError("FAILED: Class " + createFunctionDesc.getClassName()
+ + " does not implement UDF, GenericUDF, or UDAF");
+ return 1;
+ }
+
+ // TODO: There should be a registerPermanentFunction()
+ addedToRegistry = FunctionRegistry.registerTemporaryFunction(registeredName, udfClass);
+ if (!addedToRegistry) {
+ console.printError("Failed to register " + registeredName
+ + " using class " + createFunctionDesc.getClassName());
+ return 1;
+ }
+
+ // Add to metastore
+ Function func = new Function(
+ funcName,
+ dbName,
+ className,
+ SessionState.get().getUserName(),
+ PrincipalType.USER,
+ (int) (System.currentTimeMillis() / 1000),
+ org.apache.hadoop.hive.metastore.api.FunctionType.JAVA,
+ createFunctionDesc.getResources()
+ );
+ db.createFunction(func);
+ return 0;
+ } catch (ClassNotFoundException e) {
+ console.printError("FAILED: Class " + createFunctionDesc.getClassName() + " not found");
+ LOG.info("create function: " + StringUtils.stringifyException(e));
+ if (addedToRegistry) {
+ FunctionRegistry.unregisterTemporaryUDF(registeredName);
+ }
+ return 1;
+ }
+ }
+
+ private int createTemporaryFunction(CreateFunctionDesc createFunctionDesc) {
try {
+ // Add any required resources
+ addFunctionResources(createFunctionDesc.getResources());
+
Class<?> udfClass = getUdfClass(createFunctionDesc);
boolean registered = FunctionRegistry.registerTemporaryFunction(
createFunctionDesc.getFunctionName(),
@@ -94,7 +188,12 @@ public class FunctionTask extends Task<F
console.printError("FAILED: Class " + createFunctionDesc.getClassName()
+ " does not implement UDF, GenericUDF, or UDAF");
return 1;
+ } catch (HiveException e) {
+ console.printError("FAILED: " + e.toString());
+ LOG.info("create function: " + StringUtils.stringifyException(e));
+ return 1;
} catch (ClassNotFoundException e) {
+
console.printError("FAILED: Class " + createFunctionDesc.getClassName() + " not found");
LOG.info("create function: " + StringUtils.stringifyException(e));
return 1;
@@ -121,7 +220,27 @@ public class FunctionTask extends Task<F
}
}
- private int dropFunction(DropFunctionDesc dropFunctionDesc) {
+ private int dropPermanentFunction(Hive db, DropFunctionDesc dropFunctionDesc) {
+ try {
+ String[] qualifiedNameParts = FunctionUtils.getQualifiedFunctionNameParts(
+ dropFunctionDesc.getFunctionName());
+ String dbName = qualifiedNameParts[0];
+ String funcName = qualifiedNameParts[1];
+
+ String registeredName = FunctionUtils.qualifyFunctionName(funcName, dbName);
+ // TODO: there should be a unregisterPermanentUDF()
+ FunctionRegistry.unregisterTemporaryUDF(registeredName);
+ db.dropFunction(dbName, funcName);
+
+ return 0;
+ } catch (Exception e) {
+ LOG.info("drop function: " + StringUtils.stringifyException(e));
+ console.printError("FAILED: error during drop function: " + StringUtils.stringifyException(e));
+ return 1;
+ }
+ }
+
+ private int dropTemporaryFunction(DropFunctionDesc dropFunctionDesc) {
try {
FunctionRegistry.unregisterTemporaryUDF(dropFunctionDesc
.getFunctionName());
@@ -132,6 +251,63 @@ public class FunctionTask extends Task<F
}
}
+ private void checkLocalFunctionResources(Hive db, List<ResourceUri> resources)
+ throws HiveException {
+ // If this is a non-local warehouse, then adding resources from the local filesystem
+ // may mean that other clients will not be able to access the resources.
+ // So disallow resources from local filesystem in this case.
+ if (resources != null && resources.size() > 0) {
+ try {
+ String localFsScheme = FileSystem.getLocal(db.getConf()).getUri().getScheme();
+ String configuredFsScheme = FileSystem.get(db.getConf()).getUri().getScheme();
+ if (configuredFsScheme.equals(localFsScheme)) {
+ // Configured warehouse FS is local, don't need to bother checking.
+ return;
+ }
+
+ for (ResourceUri res : resources) {
+ String resUri = res.getUri();
+ if (!SessionState.canDownloadResource(resUri)) {
+ throw new HiveException("Hive warehouse is non-local, but "
+ + res.getUri() + " specifies file on local filesystem. "
+ + "Resources on non-local warehouse should specify a non-local scheme/path");
+ }
+ }
+ } catch (HiveException e) {
+ throw e;
+ } catch (Exception e) {
+ LOG.error(e);
+ throw new HiveException(e);
+ }
+ }
+ }
+
+
+ private static SessionState.ResourceType getResourceType(ResourceType rt) throws HiveException {
+ switch (rt) {
+ case JAR:
+ return SessionState.ResourceType.JAR;
+ case FILE:
+ return SessionState.ResourceType.FILE;
+ case ARCHIVE:
+ return SessionState.ResourceType.ARCHIVE;
+ default:
+ throw new HiveException("Unexpected resource type " + rt);
+ }
+ }
+
+ public static void addFunctionResources(List<ResourceUri> resources) throws HiveException {
+ if (resources != null) {
+ for (ResourceUri res : resources) {
+ String addedResource =
+ SessionState.get().add_resource(getResourceType(res.getResourceType()), res.getUri());
+ if (addedResource == null) {
+ throw new HiveException("Unable to load " + res.getResourceType() + " " + res.getUri());
+ }
+ }
+ }
+ }
+
@SuppressWarnings("unchecked")
private Class<?> getUdfClass(CreateFunctionDesc desc) throws ClassNotFoundException {
return Class.forName(desc.getClassName(), true, JavaUtils.getClassLoader());
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionUtils.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionUtils.java Tue Feb 25 07:58:52 2014
@@ -19,6 +19,11 @@
package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver;
public class FunctionUtils {
@@ -51,4 +56,55 @@ public class FunctionUtils {
return names;
}
+ public static String[] getQualifiedFunctionNameParts(String name) throws HiveException {
+ if (isQualifiedFunctionName(name)) {
+ return splitQualifiedFunctionName(name);
+ }
+ String dbName = SessionState.get().getCurrentDatabase();
+ return new String[] { dbName, name };
+ }
+
+ /**
+ * Function type, for permanent functions.
+ * Currently just JAVA, though we could support Groovy later on.
+ */
+ public enum FunctionType {
+ JAVA,
+ }
+
+ /**
+ * Enum type to describe what kind of UDF implementation class
+ */
+ public enum UDFClassType {
+ UNKNOWN,
+ UDF,
+ GENERIC_UDF,
+ GENERIC_UDTF,
+ UDAF,
+ GENERIC_UDAF_RESOLVER,
+ TABLE_FUNCTION_RESOLVER,
+ }
+
+ /**
+ * Determine the UDF class type of the class
+ * @param udfClass
+ * @return UDFClassType enum corresponding to the class type of the UDF
+ */
+ public static UDFClassType getUDFClassType(Class<?> udfClass) {
+ if (UDF.class.isAssignableFrom(udfClass)) {
+ return UDFClassType.UDF;
+ } else if (GenericUDF.class.isAssignableFrom(udfClass)) {
+ return UDFClassType.GENERIC_UDF;
+ } else if (GenericUDTF.class.isAssignableFrom(udfClass)) {
+ return UDFClassType.GENERIC_UDTF;
+ } else if (UDAF.class.isAssignableFrom(udfClass)) {
+ return UDFClassType.UDAF;
+ } else if (GenericUDAFResolver.class.isAssignableFrom(udfClass)) {
+ return UDFClassType.GENERIC_UDAF_RESOLVER;
+ } else if(TableFunctionResolver.class.isAssignableFrom(udfClass)) {
+ return UDFClassType.TABLE_FUNCTION_RESOLVER;
+ } else {
+ return UDFClassType.UNKNOWN;
+ }
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java Tue Feb 25 07:58:52 2014
@@ -310,4 +310,15 @@ public class BytesColumnVector extends C
isRepeating = true;
setRef(0, value, 0, value.length);
}
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ BytesColumnVector in = (BytesColumnVector) inputVector;
+ setVal(outElementNum, in.vector[inputElementNum], in.start[inputElementNum], in.length[outElementNum]);
+ }
+
+ @Override
+ public void init() {
+ initBuffer(0);
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java Tue Feb 25 07:58:52 2014
@@ -142,5 +142,19 @@ public abstract class ColumnVector {
preFlattenIsRepeating = isRepeating;
preFlattenNoNulls = noNulls;
}
+
+ /**
+ * Set the element in this column vector from the given input vector.
+ */
+ public abstract void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector);
+
+ /**
+ * Initialize the column vector. This method can be overridden by specific column vector types.
+ * Use this method only if the individual type of the column vector is not known, otherwise its
+ * preferable to call specific initialization methods.
+ */
+ public void init() {
+ // Do nothing by default
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java Tue Feb 25 07:58:52 2014
@@ -76,6 +76,12 @@ public class DecimalColumnVector extends
// TODO Auto-generated method stub
}
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ vector[outElementNum].update(((DecimalColumnVector) inputVector).vector[inputElementNum]);
+ vector[outElementNum].changeScaleDestructive(scale);
+ }
+
/**
* Check if the value at position i fits in the available precision,
* and convert the value to NULL if it does not.
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/DoubleColumnVector.java Tue Feb 25 07:58:52 2014
@@ -141,4 +141,9 @@ public class DoubleColumnVector extends
}
flattenNoNulls(selectedInUse, sel, size);
}
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ vector[outElementNum] = ((DoubleColumnVector) inputVector).vector[inputElementNum];
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/LongColumnVector.java Tue Feb 25 07:58:52 2014
@@ -185,4 +185,9 @@ public class LongColumnVector extends Co
}
flattenNoNulls(selectedInUse, sel, size);
}
+
+ @Override
+ public void setElement(int outElementNum, int inputElementNum, ColumnVector inputVector) {
+ vector[outElementNum] = ((LongColumnVector) inputVector).vector[inputElementNum];
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnAssignFactory.java Tue Feb 25 07:58:52 2014
@@ -23,9 +23,12 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.type.Decimal128;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -141,6 +144,23 @@ public class VectorColumnAssignFactory {
}
}
+ private static abstract class VectorDecimalColumnAssign
+ extends VectorColumnAssignVectorBase<DecimalColumnVector> {
+ protected void assignDecimal(HiveDecimal value, int index) {
+ outCol.vector[index].update(value.unscaledValue(), (byte) value.scale());
+ }
+
+ protected void assignDecimal(Decimal128 value, int index) {
+ outCol.vector[index].update(value);
+ }
+ protected void assignDecimal(HiveDecimalWritable hdw, int index) {
+ byte[] internalStorage = hdw.getInternalStorage();
+ int scale = hdw.getScale();
+
+ outCol.vector[index].fastUpdateFromInternalStorage(internalStorage, (short)scale);
+ }
+ }
+
public static VectorColumnAssign[] buildAssigners(VectorizedRowBatch outputBatch)
throws HiveException {
@@ -175,6 +195,14 @@ public class VectorColumnAssignFactory {
}
}.init(outputBatch, (BytesColumnVector) cv);
}
+ else if (cv instanceof DecimalColumnVector) {
+ vca[i] = new VectorDecimalColumnAssign() {
+ @Override
+ protected void copyValue(DecimalColumnVector src, int srcIndex, int destIndex) {
+ assignDecimal(src.vector[srcIndex], destIndex);
+ }
+ };
+ }
else {
throw new HiveException("Unimplemented vector column type: " + cv.getClass().getName());
}
@@ -336,6 +364,27 @@ public class VectorColumnAssignFactory {
poi.getPrimitiveCategory());
}
}
+ else if (destCol instanceof DecimalColumnVector) {
+ switch(poi.getPrimitiveCategory()) {
+ case DECIMAL:
+ outVCA = new VectorDecimalColumnAssign() {
+ @Override
+ public void assignObjectValue(Object val, int destIndex) throws HiveException {
+ if (val == null) {
+ assignNull(destIndex);
+ }
+ else {
+ HiveDecimalWritable hdw = (HiveDecimalWritable) val;
+ assignDecimal(hdw, destIndex);
+ }
+ }
+ }.init(outputBatch, (DecimalColumnVector) destCol);
+ break;
+ default:
+ throw new HiveException("Incompatible Decimal vector column and primitive category " +
+ poi.getPrimitiveCategory());
+ }
+ }
else {
throw new HiveException("Unknown vector column type " + destCol.getClass().getName());
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExpressionDescriptor.java Tue Feb 25 07:58:52 2014
@@ -49,7 +49,7 @@ public class VectorExpressionDescriptor
public static ArgumentType getType(String inType) {
String type = VectorizationContext.getNormalizedTypeName(inType);
- if (VectorizationContext.decimalTypePattern.matcher(type.toLowerCase()).matches()) {
+ if (VectorizationContext.decimalTypePattern.matcher(type).matches()) {
type = "decimal";
}
return valueOf(type.toUpperCase());
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java Tue Feb 25 07:58:52 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.v
import java.util.Arrays;
+import org.apache.hadoop.hive.common.type.Decimal128;
import org.apache.hadoop.hive.ql.exec.KeyWrapper;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -42,16 +43,23 @@ public class VectorHashKeyWrapper extend
private int[] byteStarts;
private int[] byteLengths;
+ private Decimal128[] decimalValues;
+
private boolean[] isNull;
private int hashcode;
- public VectorHashKeyWrapper(int longValuesCount, int doubleValuesCount, int byteValuesCount) {
+ public VectorHashKeyWrapper(int longValuesCount, int doubleValuesCount,
+ int byteValuesCount, int decimalValuesCount) {
longValues = new long[longValuesCount];
doubleValues = new double[doubleValuesCount];
+ decimalValues = new Decimal128[decimalValuesCount];
+ for(int i = 0; i < decimalValuesCount; ++i) {
+ decimalValues[i] = new Decimal128();
+ }
byteValues = new byte[byteValuesCount][];
byteStarts = new int[byteValuesCount];
byteLengths = new int[byteValuesCount];
- isNull = new boolean[longValuesCount + doubleValuesCount + byteValuesCount];
+ isNull = new boolean[longValuesCount + doubleValuesCount + byteValuesCount + decimalValuesCount];
}
private VectorHashKeyWrapper() {
@@ -66,6 +74,7 @@ public class VectorHashKeyWrapper extend
public void setHashKey() {
hashcode = Arrays.hashCode(longValues) ^
Arrays.hashCode(doubleValues) ^
+ Arrays.hashCode(decimalValues) ^
Arrays.hashCode(isNull);
// This code, with branches and all, is not executed if there are no string keys
@@ -104,6 +113,7 @@ public class VectorHashKeyWrapper extend
return hashcode == keyThat.hashcode &&
Arrays.equals(longValues, keyThat.longValues) &&
Arrays.equals(doubleValues, keyThat.doubleValues) &&
+ Arrays.equals(decimalValues, keyThat.decimalValues) &&
Arrays.equals(isNull, keyThat.isNull) &&
byteValues.length == keyThat.byteValues.length &&
(0 == byteValues.length || bytesEquals(keyThat));
@@ -137,6 +147,12 @@ public class VectorHashKeyWrapper extend
clone.doubleValues = doubleValues.clone();
clone.isNull = isNull.clone();
+ // Decimal128 requires deep clone
+ clone.decimalValues = new Decimal128[decimalValues.length];
+ for(int i = 0; i < decimalValues.length; ++i) {
+ clone.decimalValues[i] = new Decimal128().update(decimalValues[i]);
+ }
+
clone.byteValues = new byte[byteValues.length][];
clone.byteStarts = new int[byteValues.length];
clone.byteLengths = byteLengths.clone();
@@ -201,13 +217,22 @@ public class VectorHashKeyWrapper extend
isNull[longValues.length + doubleValues.length + index] = true;
}
+ public void assignDecimal(int index, Decimal128 value) {
+ decimalValues[index].update(value);
+ }
+
+ public void assignNullDecimal(int index) {
+ isNull[longValues.length + doubleValues.length + byteValues.length + index] = true;
+ }
+
@Override
public String toString()
{
- return String.format("%d[%s] %d[%s] %d[%s]",
+ return String.format("%d[%s] %d[%s] %d[%s] %d[%s]",
longValues.length, Arrays.toString(longValues),
doubleValues.length, Arrays.toString(doubleValues),
- byteValues.length, Arrays.toString(byteValues));
+ byteValues.length, Arrays.toString(byteValues),
+ decimalValues.length, Arrays.toString(decimalValues));
}
public boolean getIsLongNull(int i) {
@@ -222,7 +247,7 @@ public class VectorHashKeyWrapper extend
return isNull[longValues.length + doubleValues.length + i];
}
-
+
public long getLongValue(int i) {
return longValues[i];
}
@@ -252,6 +277,12 @@ public class VectorHashKeyWrapper extend
return variableSize;
}
+ public boolean getIsDecimalNull(int i) {
+ return isNull[longValues.length + doubleValues.length + byteValues.length + i];
+ }
+ public Decimal128 getDecimal(int i) {
+ return decimalValues[i];
+ }
}
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java Tue Feb 25 07:58:52 2014
@@ -40,9 +40,40 @@ public class VectorHashKeyWrapperBatch {
private int longIndex;
private int doubleIndex;
private int stringIndex;
+ private int decimalIndex;
+
+ private static final int INDEX_UNUSED = -1;
+
+ private void resetIndices() {
+ this.longIndex = this.doubleIndex = this.stringIndex = this.decimalIndex = INDEX_UNUSED;
+ }
+ public void setLong(int index) {
+ resetIndices();
+ this.longIndex= index;
+ }
+
+ public void setDouble(int index) {
+ resetIndices();
+ this.doubleIndex = index;
+ }
+
+ public void setString(int index) {
+ resetIndices();
+ this.stringIndex = index;
+ }
+
+ public void setDecimal(int index) {
+ resetIndices();
+ this.decimalIndex = index;
+ }
}
/**
+ * Number of object references in 'this' (for size computation)
+ */
+ private static final int MODEL_REFERENCES_COUNT = 7;
+
+ /**
* The key expressions that require evaluation and output the primitive values for each key.
*/
private VectorExpression[] keyExpressions;
@@ -63,6 +94,11 @@ public class VectorHashKeyWrapperBatch {
private int[] stringIndices;
/**
+ * indices of decimal primitive keys.
+ */
+ private int[] decimalIndices;
+
+ /**
* Pre-allocated batch size vector of keys wrappers.
* N.B. these keys are **mutable** and should never be used in a HashMap.
* Always clone the key wrapper to obtain an immutable keywrapper suitable
@@ -175,6 +211,28 @@ public class VectorHashKeyWrapperBatch {
columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
}
}
+ for(int i=0;i<decimalIndices.length; ++i) {
+ int keyIndex = decimalIndices[i];
+ int columnIndex = keyExpressions[keyIndex].getOutputColumn();
+ DecimalColumnVector columnVector = (DecimalColumnVector) batch.cols[columnIndex];
+ if (columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDecimalNoNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDecimalNoNullsNoRepeatingSelection(i, batch.size, columnVector, batch.selected);
+ } else if (columnVector.noNulls && columnVector.isRepeating) {
+ assignDecimalNoNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && !batch.selectedInUse) {
+ assignDecimalNullsNoRepeatingNoSelection(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && columnVector.isRepeating) {
+ assignDecimalNullsRepeating(i, batch.size, columnVector);
+ } else if (!columnVector.noNulls && !columnVector.isRepeating && batch.selectedInUse) {
+ assignDecimalNullsNoRepeatingSelection (i, batch.size, columnVector, batch.selected);
+ } else {
+ throw new HiveException (String.format(
+ "Unimplemented Decimal null/repeat/selected combination %b/%b/%b",
+ columnVector.noNulls, columnVector.isRepeating, batch.selectedInUse));
+ }
+ }
for(int i=0;i<batch.size;++i) {
vectorHashKeyWrappers[i].setHashKey();
}
@@ -427,6 +485,80 @@ public class VectorHashKeyWrapperBatch {
}
/**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Decimal type, possible nulls, no repeat values, batch selection vector.
+ */
+ private void assignDecimalNullsNoRepeatingSelection(int index, int size,
+ DecimalColumnVector columnVector, int[] selected) {
+ for(int i = 0; i < size; ++i) {
+ int row = selected[i];
+ if (!columnVector.isNull[row]) {
+ vectorHashKeyWrappers[i].assignDecimal(index, columnVector.vector[row]);
+ } else {
+ vectorHashKeyWrappers[i].assignNullDecimal(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Decimal type, repeat null values.
+ */
+ private void assignDecimalNullsRepeating(int index, int size,
+ DecimalColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignNullDecimal(index);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Decimal type, possible nulls, repeat values.
+ */
+ private void assignDecimalNullsNoRepeatingNoSelection(int index, int size,
+ DecimalColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ if (!columnVector.isNull[r]) {
+ vectorHashKeyWrappers[r].assignDecimal(index, columnVector.vector[r]);
+ } else {
+ vectorHashKeyWrappers[r].assignNullDecimal(index);
+ }
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Decimal type, no nulls, repeat values, no selection vector.
+ */
+ private void assignDecimalNoNullsRepeating(int index, int size, DecimalColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignDecimal(index, columnVector.vector[0]);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Decimal type, no nulls, no repeat values, batch selection vector.
+ */
+ private void assignDecimalNoNullsNoRepeatingSelection(int index, int size,
+ DecimalColumnVector columnVector, int[] selected) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignDecimal(index, columnVector.vector[selected[r]]);
+ }
+ }
+
+ /**
+ * Helper method to assign values from a vector column into the key wrapper.
+ * Optimized for Decimal type, no nulls, no repeat values, no selection vector.
+ */
+ private void assignDecimalNoNullsNoRepeatingNoSelection(int index, int size,
+ DecimalColumnVector columnVector) {
+ for(int r = 0; r < size; ++r) {
+ vectorHashKeyWrappers[r].assignDecimal(index, columnVector.vector[r]);
+ }
+ }
+
+ /**
* Prepares a VectorHashKeyWrapperBatch to work for a specific set of keys.
* Computes the fast access lookup indices, preallocates all needed internal arrays.
* This step is done only once per query, not once per batch. The information computed now
@@ -446,6 +578,8 @@ public class VectorHashKeyWrapperBatch {
int doubleIndicesIndex = 0;
int[] stringIndices = new int[keyExpressions.length];
int stringIndicesIndex = 0;
+ int[] decimalIndices = new int[keyExpressions.length];
+ int decimalIndicesIndex = 0;
KeyLookupHelper[] indexLookup = new KeyLookupHelper[keyExpressions.length];
// Inspect the output type of each key expression.
@@ -455,22 +589,20 @@ public class VectorHashKeyWrapperBatch {
if (VectorizationContext.isIntFamily(outputType) ||
VectorizationContext.isDatetimeFamily(outputType)) {
longIndices[longIndicesIndex] = i;
- indexLookup[i].longIndex = longIndicesIndex;
- indexLookup[i].doubleIndex = -1;
- indexLookup[i].stringIndex = -1;
+ indexLookup[i].setLong(longIndicesIndex);
++longIndicesIndex;
} else if (VectorizationContext.isFloatFamily(outputType)) {
doubleIndices[doubleIndicesIndex] = i;
- indexLookup[i].longIndex = -1;
- indexLookup[i].doubleIndex = doubleIndicesIndex;
- indexLookup[i].stringIndex = -1;
+ indexLookup[i].setDouble(doubleIndicesIndex);
++doubleIndicesIndex;
} else if (VectorizationContext.isStringFamily(outputType)) {
stringIndices[stringIndicesIndex]= i;
- indexLookup[i].longIndex = -1;
- indexLookup[i].doubleIndex = -1;
- indexLookup[i].stringIndex = stringIndicesIndex;
+ indexLookup[i].setString(stringIndicesIndex);
++stringIndicesIndex;
+ } else if (VectorizationContext.isDecimalFamily(outputType)) {
+ decimalIndices[decimalIndicesIndex]= i;
+ indexLookup[i].setDecimal(decimalIndicesIndex);
+ ++decimalIndicesIndex;
}
else {
throw new HiveException("Unsuported vector output type: " + outputType);
@@ -480,11 +612,13 @@ public class VectorHashKeyWrapperBatch {
compiledKeyWrapperBatch.longIndices = Arrays.copyOf(longIndices, longIndicesIndex);
compiledKeyWrapperBatch.doubleIndices = Arrays.copyOf(doubleIndices, doubleIndicesIndex);
compiledKeyWrapperBatch.stringIndices = Arrays.copyOf(stringIndices, stringIndicesIndex);
+ compiledKeyWrapperBatch.decimalIndices = Arrays.copyOf(decimalIndices, decimalIndicesIndex);
compiledKeyWrapperBatch.vectorHashKeyWrappers =
new VectorHashKeyWrapper[VectorizedRowBatch.DEFAULT_SIZE];
for(int i=0;i<VectorizedRowBatch.DEFAULT_SIZE; ++i) {
compiledKeyWrapperBatch.vectorHashKeyWrappers[i] =
- new VectorHashKeyWrapper(longIndicesIndex, doubleIndicesIndex, stringIndicesIndex);
+ new VectorHashKeyWrapper(longIndicesIndex, doubleIndicesIndex,
+ stringIndicesIndex, decimalIndicesIndex);
}
JavaDataModel model = JavaDataModel.get();
@@ -493,7 +627,7 @@ public class VectorHashKeyWrapperBatch {
// start with the keywrapper itself
compiledKeyWrapperBatch.keysFixedSize += JavaDataModel.alignUp(
model.object() +
- model.ref() * 6+
+ model.ref() * MODEL_REFERENCES_COUNT +
model.primitive1(),
model.memoryAlign());
@@ -501,6 +635,7 @@ public class VectorHashKeyWrapperBatch {
compiledKeyWrapperBatch.keysFixedSize += model.lengthForLongArrayOfSize(longIndicesIndex);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForDoubleArrayOfSize(doubleIndicesIndex);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(stringIndicesIndex);
+ compiledKeyWrapperBatch.keysFixedSize += model.lengthForObjectArrayOfSize(decimalIndicesIndex);
compiledKeyWrapperBatch.keysFixedSize += model.lengthForIntArrayOfSize(longIndicesIndex) * 2;
compiledKeyWrapperBatch.keysFixedSize +=
model.lengthForBooleanArrayOfSize(keyExpressions.length);
@@ -529,7 +664,12 @@ public class VectorHashKeyWrapperBatch {
kw.getBytes(klh.stringIndex),
kw.getByteStart(klh.stringIndex),
kw.getByteLength(klh.stringIndex));
- } else {
+ } else if (klh.decimalIndex >= 0) {
+ return kw.getIsDecimalNull(klh.decimalIndex)? null :
+ keyOutputWriter.writeValue(
+ kw.getDecimal(klh.decimalIndex));
+ }
+ else {
throw new HiveException(String.format(
"Internal inconsistent KeyLookupHelper at index [%d]:%d %d %d",
i, klh.longIndex, klh.doubleIndex, klh.stringIndex));
Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java?rev=1571600&r1=1571599&r2=1571600&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapJoinOperator.java Tue Feb 25 07:58:52 2014
@@ -63,8 +63,8 @@ public class VectorMapJoinOperator exten
private int tagLen;
private VectorExpression[] keyExpressions;
- private VectorHashKeyWrapperBatch keyWrapperBatch;
- private VectorExpressionWriter[] keyOutputWriters;
+ private transient VectorHashKeyWrapperBatch keyWrapperBatch;
+ private transient VectorExpressionWriter[] keyOutputWriters;
private VectorExpression[] bigTableFilterExpressions;
private VectorExpression[] bigTableValueExpressions;
@@ -111,7 +111,6 @@ public class VectorMapJoinOperator exten
List<ExprNodeDesc> keyDesc = desc.getKeys().get(posBigTable);
keyExpressions = vContext.getVectorExpressions(keyDesc);
- keyOutputWriters = VectorExpressionWriterFactory.getExpressionWriters(keyDesc);
// We're only going to evaluate the big table vectorized expressions,
Map<Byte, List<ExprNodeDesc>> exprs = desc.getExprs();
@@ -135,6 +134,8 @@ public class VectorMapJoinOperator exten
public void initializeOp(Configuration hconf) throws HiveException {
super.initializeOp(hconf);
+ List<ExprNodeDesc> keyDesc = conf.getKeys().get(posBigTable);
+ keyOutputWriters = VectorExpressionWriterFactory.getExpressionWriters(keyDesc);
vrbCtx = new VectorizedRowBatchCtx();
vrbCtx.init(hconf, this.fileKey, (StructObjectInspector) this.outputObjInspector);