You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/05/08 06:43:18 UTC
svn commit: r1480158 [2/2] - in /hive/branches/HIVE-4115: ./ common/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/ hcatalog/
hcatalog/server-extensions/src/main/java/org/apache/hcatalog/listener/
hcatalog/src/test/e2e/templeton/drivers...
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCount.java Wed May 8 04:43:16 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.De
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -99,8 +100,11 @@ public class GenericUDAFCount implements
}
/** class for storing count value. */
- static class CountAgg implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class CountAgg extends AbstractAggregationBuffer {
long value;
+ @Override
+ public int estimate() { return JavaDataModel.PRIMITIVES2; }
}
@Override
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFCovariance.java Wed May 8 04:43:16 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.De
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -38,7 +39,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.util.StringUtils;
/**
* Compute the covariance covar_pop(x, y), using the following one-pass method
@@ -224,11 +224,14 @@ public class GenericUDAFCovariance exten
}
}
- static class StdAgg implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class StdAgg extends AbstractAggregationBuffer {
long count; // number n of elements
double xavg; // average of x elements
double yavg; // average of y elements
double covar; // n times the covariance
+ @Override
+ public int estimate() { return JavaDataModel.PRIMITIVES2 * 4; }
};
@Override
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEWAHBitmap.java Wed May 8 04:43:16 2013
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.udf.ge
import java.io.IOException;
import java.util.ArrayList;
-import java.util.List;
import javaewah.EWAHCompressedBitmap;
@@ -109,8 +108,13 @@ public class GenericUDAFEWAHBitmap exten
}
/** class for storing the current partial result aggregation */
- static class BitmapAgg implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class BitmapAgg extends AbstractAggregationBuffer {
EWAHCompressedBitmap bitmap;
+ @Override
+ public int estimate() {
+ return bitmap.sizeInBytes();
+ }
}
@Override
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java Wed May 8 04:43:16 2013
@@ -41,6 +41,19 @@ import org.apache.hadoop.hive.serde2.obj
@UDFType(deterministic = true)
public abstract class GenericUDAFEvaluator implements Closeable {
+ public static @interface AggregationType {
+ boolean estimable() default false;
+ }
+
+ public static boolean isEstimable(AggregationBuffer buffer) {
+ if (buffer instanceof AbstractAggregationBuffer) {
+ Class<? extends AggregationBuffer> clazz = buffer.getClass();
+ AggregationType annotation = clazz.getAnnotation(AggregationType.class);
+ return annotation != null && annotation.estimable();
+ }
+ return false;
+ }
+
/**
* Mode.
*
@@ -123,10 +136,21 @@ public abstract class GenericUDAFEvaluat
*
* In the future, we may completely hide this class inside the Evaluator and
* use integer numbers to identify which aggregation we are looking at.
+ *
+ * @deprecated use {@link AbstractAggregationBuffer} instead
*/
public static interface AggregationBuffer {
};
+ public static abstract class AbstractAggregationBuffer implements AggregationBuffer {
+ /**
+ * Estimate the size of memory which is occupied by aggregation buffer.
+ * Currently, hive assumes that primitives types occupies 16 byte and java object has
+ * 64 byte overhead for each. For map, each entry also has 64 byte overhead.
+ */
+ public int estimate() { return -1; }
+ }
+
/**
* Get a new aggregation object.
*/
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFHistogramNumeric.java Wed May 8 04:43:16 2013
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.udf.ge
import java.util.ArrayList;
import java.util.List;
-import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,21 +26,16 @@ import org.apache.hadoop.hive.ql.exec.De
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.util.StringUtils;
/**
* Computes an approximate histogram of a numerical column using a user-specified number of bins.
@@ -235,8 +229,13 @@ public class GenericUDAFHistogramNumeric
// Aggregation buffer definition and manipulation methods
- static class StdAgg implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class StdAgg extends AbstractAggregationBuffer {
NumericHistogram histogram; // the histogram object
+ @Override
+ public int estimate() {
+ return JavaDataModel.get().lengthFor(histogram);
+ }
};
@Override
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMax.java Wed May 8 04:43:16 2013
@@ -71,7 +71,7 @@ public class GenericUDAFMax extends Abst
}
/** class for storing the current max value */
- static class MaxAgg implements AggregationBuffer {
+ static class MaxAgg extends AbstractAggregationBuffer {
Object o;
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFMin.java Wed May 8 04:43:16 2013
@@ -71,7 +71,7 @@ public class GenericUDAFMin extends Abst
}
/** class for storing the current max value */
- static class MinAgg implements AggregationBuffer {
+ static class MinAgg extends AbstractAggregationBuffer {
Object o;
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFPercentileApprox.java Wed May 8 04:43:16 2013
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.exec.De
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
@@ -33,15 +34,9 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.util.StringUtils;
/**
* Computes an approximate percentile (quantile) from an approximate histogram, for very
@@ -353,9 +348,16 @@ public class GenericUDAFPercentileApprox
// Aggregation buffer methods. We wrap GenericUDAFHistogramNumeric's aggregation buffer
// inside our own, so that we can also store requested quantile values between calls
- static class PercentileAggBuf implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class PercentileAggBuf extends AbstractAggregationBuffer {
NumericHistogram histogram; // histogram used for quantile approximation
double[] quantiles; // the quantiles requested
+ @Override
+ public int estimate() {
+ JavaDataModel model = JavaDataModel.get();
+ return model.lengthFor(histogram) +
+ model.array() + JavaDataModel.PRIMITIVES2 * quantiles.length;
+ }
};
@Override
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java Wed May 8 04:43:16 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.hive.ql.exec.De
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -96,7 +97,8 @@ public class GenericUDAFSum extends Abst
}
/** class for storing decimal sum value. */
- static class SumHiveDecimalAgg implements AggregationBuffer {
+ @AggregationType(estimable = false) // hard to know exactly for decimals
+ static class SumHiveDecimalAgg extends AbstractAggregationBuffer {
boolean empty;
HiveDecimal sum;
}
@@ -188,9 +190,12 @@ public class GenericUDAFSum extends Abst
}
/** class for storing double sum value. */
- static class SumDoubleAgg implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class SumDoubleAgg extends AbstractAggregationBuffer {
boolean empty;
double sum;
+ @Override
+ public int estimate() { return JavaDataModel.PRIMITIVES1 + JavaDataModel.PRIMITIVES2; }
}
@Override
@@ -270,9 +275,12 @@ public class GenericUDAFSum extends Abst
}
/** class for storing double sum value. */
- static class SumLongAgg implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class SumLongAgg extends AbstractAggregationBuffer {
boolean empty;
long sum;
+ @Override
+ public int estimate() { return JavaDataModel.PRIMITIVES1 + JavaDataModel.PRIMITIVES2; }
}
@Override
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFVariance.java Wed May 8 04:43:16 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.De
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
@@ -171,10 +172,13 @@ public class GenericUDAFVariance extends
}
}
- static class StdAgg implements AggregationBuffer {
+ @AggregationType(estimable = true)
+ static class StdAgg extends AbstractAggregationBuffer {
long count; // number of elements
double sum; // sum of elements
double variance; // sum[x-avg^2] (this is actually n times the variance)
+ @Override
+ public int estimate() { return JavaDataModel.PRIMITIVES2 * 3; }
};
@Override
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java Wed May 8 04:43:16 2013
@@ -338,7 +338,7 @@ public class GenericUDAFnGrams implement
}
// Aggregation buffer methods.
- static class NGramAggBuf implements AggregationBuffer {
+ static class NGramAggBuf extends AbstractAggregationBuffer {
NGramEstimator nge;
int n;
};
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/NumericHistogram.java Wed May 8 04:43:16 2013
@@ -313,4 +313,8 @@ public class NumericHistogram {
return result;
}
+
+ public int getNumBins() {
+ return bins == null ? 0 : bins.size();
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/column_access_stats.q
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/column_access_stats.q?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/column_access_stats.q (original)
+++ hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/column_access_stats.q Wed May 8 04:43:16 2013
@@ -8,11 +8,15 @@ LOAD DATA LOCAL INPATH '../data/files/T1
CREATE TABLE T2(key STRING, val STRING) STORED AS TEXTFILE;
CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE;
+CREATE TABLE T4(key STRING, val STRING) PARTITIONED BY (p STRING);
-- Simple select queries
SELECT key FROM T1 ORDER BY key;
SELECT key, val FROM T1 ORDER BY key, val;
SELECT 1 FROM T1;
+SELECT key, val from T4 where p=1;
+SELECT val FROM T4 where p=1;
+SELECT p, val FROM T4 where p=1;
-- More complicated select queries
EXPLAIN SELECT key FROM (SELECT key, val FROM T1) subq1 ORDER BY key;
Modified: hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/input4.q
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/input4.q?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/input4.q (original)
+++ hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/input4.q Wed May 8 04:43:16 2013
@@ -3,6 +3,6 @@ EXPLAIN
LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4;
LOAD DATA LOCAL INPATH '../data/files/kv1.txt' INTO TABLE INPUT4;
EXPLAIN FORMATTED
-SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4;
-SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4;
+SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias;
+SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias
Modified: hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join39.q
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join39.q?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join39.q (original)
+++ hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join39.q Wed May 8 04:43:16 2013
@@ -1,7 +1,3 @@
-set hive.mapjoin.cache.numrows = 2;
-
-
-
CREATE TABLE dest_j1(key STRING, value STRING, key1 string, val2 STRING) STORED AS TEXTFILE;
explain
Modified: hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join40.q
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join40.q?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join40.q (original)
+++ hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/join40.q Wed May 8 04:43:16 2013
@@ -29,7 +29,6 @@ SORT BY src1.key, src1.value, src2.key,
SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key < 15) RIGHT OUTER JOIN src src3 ON (src1.key = src3.key AND src3.key < 20)
SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value;
-set hive.mapjoin.cache.numrows=2;
EXPLAIN
SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value
Modified: hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/mapjoin1.q
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/mapjoin1.q?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/mapjoin1.q (original)
+++ hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/mapjoin1.q Wed May 8 04:43:16 2013
@@ -1,5 +1,3 @@
-set hive.mapjoin.cache.numrows=100;
-
SELECT /*+ MAPJOIN(b) */ sum(a.key) as sum_a
FROM srcpart a
JOIN src b ON a.key = b.key where a.ds is not null;
Modified: hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join39.q.out
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join39.q.out?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join39.q.out (original)
+++ hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join39.q.out Wed May 8 04:43:16 2013
@@ -1,10 +1,5 @@
Saving all output to "!!{outputDirectory}!!/join39.q.raw". Enter "record" with no arguments to stop it.
>>> !run !!{qFileDirectory}!!/join39.q
->>> set hive.mapjoin.cache.numrows = 2;
-No rows affected
->>>
->>>
->>>
>>> CREATE TABLE dest_j1(key STRING, value STRING, key1 string, val2 STRING) STORED AS TEXTFILE;
No rows affected
>>>
Modified: hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join40.q.out
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join40.q.out?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join40.q.out (original)
+++ hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/join40.q.out Wed May 8 04:43:16 2013
@@ -3205,8 +3205,6 @@ SORT BY src1.key, src1.value, src2.key,
'9','val_9','9','val_9','9','val_9'
548 rows selected
>>>
->>> set hive.mapjoin.cache.numrows=2;
-No rows affected
>>>
>>> EXPLAIN
SELECT /*+ MAPJOIN(y) */ x.key, x.value, y.key, y.value
Modified: hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/mapjoin1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/mapjoin1.q.out?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/mapjoin1.q.out (original)
+++ hive/branches/HIVE-4115/ql/src/test/results/beelinepositive/mapjoin1.q.out Wed May 8 04:43:16 2013
@@ -1,8 +1,5 @@
Saving all output to "!!{outputDirectory}!!/mapjoin1.q.raw". Enter "record" with no arguments to stop it.
>>> !run !!{qFileDirectory}!!/mapjoin1.q
->>> set hive.mapjoin.cache.numrows=100;
-No rows affected
->>>
>>> SELECT /*+ MAPJOIN(b) */ sum(a.key) as sum_a
FROM srcpart a
JOIN src b ON a.key = b.key where a.ds is not null;
Modified: hive/branches/HIVE-4115/ql/src/test/results/clientpositive/column_access_stats.q.out
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/results/clientpositive/column_access_stats.q.out?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/results/clientpositive/column_access_stats.q.out (original)
+++ hive/branches/HIVE-4115/ql/src/test/results/clientpositive/column_access_stats.q.out Wed May 8 04:43:16 2013
@@ -9,6 +9,8 @@ PREHOOK: query: CREATE TABLE T2(key STRI
PREHOOK: type: CREATETABLE
PREHOOK: query: CREATE TABLE T3(key STRING, val STRING) STORED AS TEXTFILE
PREHOOK: type: CREATETABLE
+PREHOOK: query: CREATE TABLE T4(key STRING, val STRING) PARTITIONED BY (p STRING)
+PREHOOK: type: CREATETABLE
PREHOOK: query: -- Simple select queries
SELECT key FROM T1 ORDER BY key
PREHOOK: type: QUERY
@@ -46,6 +48,27 @@ PREHOOK: Input: default@t1
1
1
1
+PREHOOK: query: SELECT key, val from T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:key,val
+
+PREHOOK: query: SELECT val FROM T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:val
+
+PREHOOK: query: SELECT p, val FROM T4 where p=1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t4
+#### A masked pattern was here ####
+Table:default@t4
+Columns:val
+
PREHOOK: query: -- More complicated select queries
EXPLAIN SELECT key FROM (SELECT key, val FROM T1) subq1 ORDER BY key
PREHOOK: type: QUERY
Modified: hive/branches/HIVE-4115/ql/src/test/results/clientpositive/input4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/results/clientpositive/input4.q.out?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/results/clientpositive/input4.q.out (original)
+++ hive/branches/HIVE-4115/ql/src/test/results/clientpositive/input4.q.out Wed May 8 04:43:16 2013
@@ -43,17 +43,17 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH
POSTHOOK: type: LOAD
POSTHOOK: Output: default@input4
PREHOOK: query: EXPLAIN FORMATTED
-SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
+SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN FORMATTED
-SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
+SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias
POSTHOOK: type: QUERY
-{"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Split Sample:":{},"Alias -> Map Operator Tree:":{"input4":{"TS_0":{"SEL_1":{"FS_2":{"File Output Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"ROOT STAGE":"TRUE"}},"ABSTRACT SYNTAX TREE":"(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME INPUT4))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL INPUT4) VALUE)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL INPUT4) KEY)))))"}
-PREHOOK: query: SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
+{"STAGE PLANS":{"Stage-1":{"Map Reduce":{"Split Sample:":{},"Alias -> Map Operator Tree:":{"input4alias":{"TS_0":{"SEL_1":{"FS_2":{"File Output Operator":{"GlobalTableId:":"0","compressed:":"false","table:":{"input format:":"org.apache.hadoop.mapred.TextInputFormat","output format:":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"}}}}}}}}},"Stage-0":{"Fetch Operator":{"limit:":"-1"}}},"STAGE DEPENDENCIES":{"Stage-1":{"ROOT STAGE":"TRUE"},"Stage-0":{"ROOT STAGE":"TRUE"}},"ABSTRACT SYNTAX TREE":"(TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME INPUT4) Input4Alias)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL Input4Alias) VALUE)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL Input4Alias) KEY)))))"}
+PREHOOK: query: SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias
PREHOOK: type: QUERY
PREHOOK: Input: default@input4
#### A masked pattern was here ####
-POSTHOOK: query: SELECT INPUT4.VALUE, INPUT4.KEY FROM INPUT4
+POSTHOOK: query: SELECT Input4Alias.VALUE, Input4Alias.KEY FROM INPUT4 AS Input4Alias
POSTHOOK: type: QUERY
POSTHOOK: Input: default@input4
#### A masked pattern was here ####
Modified: hive/branches/HIVE-4115/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/results/clientpositive/show_functions.q.out?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/branches/HIVE-4115/ql/src/test/results/clientpositive/show_functions.q.out Wed May 8 04:43:16 2013
@@ -110,7 +110,10 @@ month
named_struct
negative
ngrams
+noop
+noopwithmap
not
+npath
ntile
nvl
or
@@ -174,6 +177,7 @@ var_samp
variance
weekofyear
when
+windowingtablefunction
xpath
xpath_boolean
xpath_double
Modified: hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1480158&r1=1480157&r2=1480158&view=diff
==============================================================================
--- hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Wed May 8 04:43:16 2013
@@ -19,14 +19,17 @@
package org.apache.hive.service.cli.session;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.GetInfoType;
import org.apache.hive.service.cli.GetInfoValue;
@@ -64,6 +67,7 @@ public class HiveSessionImpl implements
private SessionManager sessionManager;
private OperationManager operationManager;
private IMetaStoreClient metastoreClient = null;
+ private final Set<OperationHandle> opHandleSet = new HashSet<OperationHandle>();
public HiveSessionImpl(String username, String password, Map<String, String> sessionConf) {
this.username = username;
@@ -141,7 +145,7 @@ public class HiveSessionImpl implements
case CLI_DBMS_NAME:
return new GetInfoValue("Apache Hive");
case CLI_DBMS_VER:
- return new GetInfoValue("0.10.0");
+ return new GetInfoValue(HiveVersionInfo.getVersion());
case CLI_MAX_COLUMN_NAME_LEN:
return new GetInfoValue(128);
case CLI_MAX_SCHEMA_NAME_LEN:
@@ -164,7 +168,9 @@ public class HiveSessionImpl implements
ExecuteStatementOperation operation = getOperationManager()
.newExecuteStatementOperation(getSession(), statement, confOverlay);
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -176,7 +182,9 @@ public class HiveSessionImpl implements
try {
GetTypeInfoOperation operation = getOperationManager().newGetTypeInfoOperation(getSession());
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -188,7 +196,9 @@ public class HiveSessionImpl implements
try {
GetCatalogsOperation operation = getOperationManager().newGetCatalogsOperation(getSession());
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -201,7 +211,9 @@ public class HiveSessionImpl implements
GetSchemasOperation operation =
getOperationManager().newGetSchemasOperation(getSession(), catalogName, schemaName);
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -215,7 +227,9 @@ public class HiveSessionImpl implements
MetadataOperation operation =
getOperationManager().newGetTablesOperation(getSession(), catalogName, schemaName, tableName, tableTypes);
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -227,7 +241,9 @@ public class HiveSessionImpl implements
try {
GetTableTypesOperation operation = getOperationManager().newGetTableTypesOperation(getSession());
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -240,7 +256,9 @@ public class HiveSessionImpl implements
GetColumnsOperation operation = getOperationManager().newGetColumnsOperation(getSession(),
catalogName, schemaName, tableName, columnName);
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -253,7 +271,9 @@ public class HiveSessionImpl implements
GetFunctionsOperation operation = getOperationManager()
.newGetFunctionsOperation(getSession(), catalogName, schemaName, functionName);
operation.run();
- return operation.getHandle();
+ OperationHandle opHandle = operation.getHandle();
+ opHandleSet.add(opHandle);
+ return opHandle;
} finally {
release();
}
@@ -270,6 +290,11 @@ public class HiveSessionImpl implements
if (metastoreClient != null) {
metastoreClient.close();
}
+ // Iterate through the opHandles and close their operations
+ for (OperationHandle opHandle : opHandleSet) {
+ operationManager.closeOperation(opHandle);
+ }
+ opHandleSet.clear();
} finally {
release();
}
@@ -300,7 +325,8 @@ public class HiveSessionImpl implements
public void closeOperation(OperationHandle opHandle) throws HiveSQLException {
acquire();
try {
- sessionManager.getOperationManager().closeOperation(opHandle);
+ operationManager.closeOperation(opHandle);
+ opHandleSet.remove(opHandle);
} finally {
release();
}
@@ -341,4 +367,4 @@ public class HiveSessionImpl implements
protected HiveSession getSession() {
return this;
}
-}
+}
\ No newline at end of file