You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2012/11/07 05:55:04 UTC
svn commit: r1406465 [13/15] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/
metastore/if/ metastore/src/gen/thrift/gen-cpp/
metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/
metastore/src/gen/...
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java?rev=1406465&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java Wed Nov 7 04:55:00 2012
@@ -0,0 +1,384 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
+import org.apache.hadoop.hive.ql.plan.FetchWork;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * ColumnStatsTask implementation.
+ **/
+
+public class ColumnStatsTask extends Task<ColumnStatsWork> implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private FetchOperator ftOp;
+ private static transient final Log LOG = LogFactory.getLog(ColumnStatsTask.class);
+
+ public ColumnStatsTask() {
+ super();
+ }
+
+ @Override
+ public void initialize(HiveConf conf, QueryPlan queryPlan, DriverContext ctx) {
+ super.initialize(conf, queryPlan, ctx);
+ try {
+ JobConf job = new JobConf(conf, ExecDriver.class);
+ ftOp = new FetchOperator(work.getfWork(), job);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void unpackBooleanStats(ObjectInspector oi, Object o, String fName,
+ ColumnStatisticsObj statsObj) {
+ long v = ((LongObjectInspector) oi).get(o);
+ if (fName.equals("counttrues")) {
+ statsObj.getStatsData().getBooleanStats().setNumTrues(v);
+ } else if (fName.equals("countfalses")) {
+ statsObj.getStatsData().getBooleanStats().setNumFalses(v);
+ } else if (fName.equals("countnulls")) {
+ statsObj.getStatsData().getBooleanStats().setNumNulls(v);
+ }
+ }
+
+ private void unpackDoubleStats(ObjectInspector oi, Object o, String fName,
+ ColumnStatisticsObj statsObj) {
+ if (fName.equals("countnulls")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getDoubleStats().setNumNulls(v);
+ } else if (fName.equals("numdistinctvalues")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getDoubleStats().setNumDVs(v);
+ } else if (fName.equals("max")) {
+ double d = ((DoubleObjectInspector) oi).get(o);
+ statsObj.getStatsData().getDoubleStats().setHighValue(d);
+ } else if (fName.equals("min")) {
+ double d = ((DoubleObjectInspector) oi).get(o);
+ statsObj.getStatsData().getDoubleStats().setLowValue(d);
+ }
+ }
+
+ private void unpackLongStats(ObjectInspector oi, Object o, String fName,
+ ColumnStatisticsObj statsObj) {
+ if (fName.equals("countnulls")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getLongStats().setNumNulls(v);
+ } else if (fName.equals("numdistinctvalues")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getLongStats().setNumDVs(v);
+ } else if (fName.equals("max")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getLongStats().setHighValue(v);
+ } else if (fName.equals("min")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getLongStats().setLowValue(v);
+ }
+ }
+
+ private void unpackStringStats(ObjectInspector oi, Object o, String fName,
+ ColumnStatisticsObj statsObj) {
+ if (fName.equals("countnulls")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getStringStats().setNumNulls(v);
+ } else if (fName.equals("numdistinctvalues")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getStringStats().setNumDVs(v);
+ } else if (fName.equals("avglength")) {
+ double d = ((DoubleObjectInspector) oi).get(o);
+ statsObj.getStatsData().getStringStats().setAvgColLen(d);
+ } else if (fName.equals("maxlength")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getStringStats().setMaxColLen(v);
+ }
+ }
+
+ private void unpackBinaryStats(ObjectInspector oi, Object o, String fName,
+ ColumnStatisticsObj statsObj) {
+ if (fName.equals("countnulls")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getBinaryStats().setNumNulls(v);
+ } else if (fName.equals("avglength")) {
+ double d = ((DoubleObjectInspector) oi).get(o);
+ statsObj.getStatsData().getBinaryStats().setAvgColLen(d);
+ } else if (fName.equals("maxlength")) {
+ long v = ((LongObjectInspector) oi).get(o);
+ statsObj.getStatsData().getBinaryStats().setMaxColLen(v);
+ }
+ }
+
+ private void unpackPrimitiveObject (ObjectInspector oi, Object o, String fieldName,
+ ColumnStatisticsObj statsObj) {
+ // First infer the type of object
+ if (fieldName.equals("columntype")) {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ String s = ((StringObjectInspector) poi).getPrimitiveJavaObject(o);
+ ColumnStatisticsData statsData = new ColumnStatisticsData();
+
+ if (s.equalsIgnoreCase("long")) {
+ LongColumnStatsData longStats = new LongColumnStatsData();
+ statsData.setLongStats(longStats);
+ statsObj.setStatsData(statsData);
+ } else if (s.equalsIgnoreCase("double")) {
+ DoubleColumnStatsData doubleStats = new DoubleColumnStatsData();
+ statsData.setDoubleStats(doubleStats);
+ statsObj.setStatsData(statsData);
+ } else if (s.equalsIgnoreCase("string")) {
+ StringColumnStatsData stringStats = new StringColumnStatsData();
+ statsData.setStringStats(stringStats);
+ statsObj.setStatsData(statsData);
+ } else if (s.equalsIgnoreCase("boolean")) {
+ BooleanColumnStatsData booleanStats = new BooleanColumnStatsData();
+ statsData.setBooleanStats(booleanStats);
+ statsObj.setStatsData(statsData);
+ } else if (s.equalsIgnoreCase("binary")) {
+ BinaryColumnStatsData binaryStats = new BinaryColumnStatsData();
+ statsData.setBinaryStats(binaryStats);
+ statsObj.setStatsData(statsData);
+ }
+ } else {
+ // invoke the right unpack method depending on data type of the column
+ if (statsObj.getStatsData().isSetBooleanStats()) {
+ unpackBooleanStats(oi, o, fieldName, statsObj);
+ } else if (statsObj.getStatsData().isSetLongStats()) {
+ unpackLongStats(oi, o, fieldName, statsObj);
+ } else if (statsObj.getStatsData().isSetDoubleStats()) {
+ unpackDoubleStats(oi,o,fieldName, statsObj);
+ } else if (statsObj.getStatsData().isSetStringStats()) {
+ unpackStringStats(oi, o, fieldName, statsObj);
+ } else if (statsObj.getStatsData().isSetBinaryStats()) {
+ unpackBinaryStats(oi, o, fieldName, statsObj);
+ }
+ }
+ }
+
+ private void unpackStructObject(ObjectInspector oi, Object o, String fName,
+ ColumnStatisticsObj cStatsObj) {
+ if (oi.getCategory() != ObjectInspector.Category.STRUCT) {
+ throw new RuntimeException("Invalid object datatype : " + oi.getCategory().toString());
+ }
+
+ StructObjectInspector soi = (StructObjectInspector) oi;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+ List<Object> list = soi.getStructFieldsDataAsList(o);
+
+ for (int i = 0; i < fields.size(); i++) {
+ // Get the field objectInspector, fieldName and the field object.
+ ObjectInspector foi = fields.get(i).getFieldObjectInspector();
+ Object f = (list == null ? null : list.get(i));
+ String fieldName = fields.get(i).getFieldName();
+
+ if (foi.getCategory() == ObjectInspector.Category.PRIMITIVE) {
+ unpackPrimitiveObject(foi, f, fieldName, cStatsObj);
+ } else {
+ unpackStructObject(foi, f, fieldName, cStatsObj);
+ }
+ }
+ }
+
+ private ColumnStatistics constructColumnStatsFromPackedRow(ObjectInspector oi,
+ Object o) throws HiveException {
+ if (oi.getCategory() != ObjectInspector.Category.STRUCT) {
+ throw new HiveException("Unexpected object type encountered while unpacking row");
+ }
+
+ String dbName = db.getCurrentDatabase();
+ String tableName = work.getColStats().getTableName();
+ String partName = null;
+ List<String> colName = work.getColStats().getColName();
+ List<String> colType = work.getColStats().getColType();
+ boolean isTblLevel = work.getColStats().isTblLevel();
+
+ if (!isTblLevel) {
+ partName = work.getColStats().getPartName();
+ }
+
+ ColumnStatisticsDesc statsDesc = getColumnStatsDesc(dbName, tableName, partName, isTblLevel);
+
+ List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>();
+ StructObjectInspector soi = (StructObjectInspector) oi;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+ List<Object> list = soi.getStructFieldsDataAsList(o);
+
+ for (int i = 0; i < fields.size(); i++) {
+ // Get the field objectInspector, fieldName and the field object.
+ ObjectInspector foi = fields.get(i).getFieldObjectInspector();
+ Object f = (list == null ? null : list.get(i));
+ String fieldName = fields.get(i).getFieldName();
+ ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
+ statsObj.setColName(colName.get(i));
+ statsObj.setColType(colType.get(i));
+ unpackStructObject(foi, f, fieldName, statsObj);
+ statsObjs.add(statsObj);
+ }
+
+ ColumnStatistics colStats = new ColumnStatistics();
+ colStats.setStatsDesc(statsDesc);
+ colStats.setStatsObj(statsObjs);
+ return colStats;
+ }
+
+ private ColumnStatisticsDesc getColumnStatsDesc(String dbName, String tableName,
+ String partName, boolean isTblLevel)
+ {
+ ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
+ statsDesc.setDbName(dbName);
+ statsDesc.setTableName(tableName);
+ statsDesc.setIsTblLevel(isTblLevel);
+
+ if (!isTblLevel) {
+ statsDesc.setPartName(partName);
+ } else {
+ statsDesc.setPartName(null);
+ }
+ return statsDesc;
+ }
+
+ private int persistPartitionStats() throws HiveException {
+ InspectableObject io = null;
+ // Fetch result of the analyze table .. compute statistics for columns ..
+ try {
+ io = fetchColumnStats();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (CommandNeedRetryException e) {
+ e.printStackTrace();
+ }
+
+ // Construct a column statistics object from the result
+ ColumnStatistics colStats = constructColumnStatsFromPackedRow(io.oi, io.o);
+
+ // Persist the column statistics object to the metastore
+ try {
+ db.updatePartitionColumnStatistics(colStats);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 0;
+ }
+
+ private int persistTableStats() throws HiveException {
+ InspectableObject io = null;
+ // Fetch result of the analyze table .. compute statistics for columns ..
+ try {
+ io = fetchColumnStats();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (CommandNeedRetryException e) {
+ e.printStackTrace();
+ }
+
+ // Construct a column statistics object from the result
+ ColumnStatistics colStats = constructColumnStatsFromPackedRow(io.oi, io.o);
+
+ // Persist the column statistics object to the metastore
+ try {
+ db.updateTableColumnStatistics(colStats);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 0;
+ }
+
+ @Override
+ public int execute(DriverContext driverContext) {
+ try {
+ if (work.getColStats().isTblLevel()) {
+ return persistTableStats();
+ } else {
+ return persistPartitionStats();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return 1;
+ }
+
+ private InspectableObject fetchColumnStats() throws IOException, CommandNeedRetryException {
+ try {
+ InspectableObject io = ftOp.getNextRow();
+ if (io == null) {
+ throw new CommandNeedRetryException();
+ }
+ return io;
+ } catch (CommandNeedRetryException e) {
+ throw e;
+ } catch (IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public StageType getType() {
+ return StageType.COLUMNSTATS;
+ }
+
+ @Override
+ public String getName() {
+ return "COLUMNSTATS TASK";
+ }
+
+ @Override
+ protected void localizeMRTmpFilesImpl(Context ctx) {
+ FetchWork fWork = work.getfWork();
+ String s = fWork.getTblDir();
+ if ((s != null) && ctx.isMRTmpFileURI(s)) {
+ fWork.setTblDir(ctx.localizeMRTmpFileURI(s));
+ }
+ ArrayList<String> ls = fWork.getPartDir();
+ if (ls != null) {
+ ctx.localizePaths(ls);
+ }
+ }
+}
Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Wed Nov 7 04:55:00 2012
@@ -129,6 +129,7 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFAverage;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFBridge;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCollectSet;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFComputeStats;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFContextNGrams;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCorrelation;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFCount;
@@ -158,7 +159,6 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCoalesce;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFConcatWS;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSortArray;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapEmpty;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapOr;
@@ -169,8 +169,8 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInFile;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInstr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLocate;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFMap;
@@ -193,6 +193,7 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFReflect;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSentences;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSize;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSortArray;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSplit;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStringToMap;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
@@ -227,10 +228,8 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ReflectionUtils;
-
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
/**
@@ -410,7 +409,6 @@ public final class FunctionRegistry {
registerGenericUDAF("sum", new GenericUDAFSum());
registerGenericUDAF("count", new GenericUDAFCount());
registerGenericUDAF("avg", new GenericUDAFAverage());
-
registerGenericUDAF("std", new GenericUDAFStd());
registerGenericUDAF("stddev", new GenericUDAFStd());
registerGenericUDAF("stddev_pop", new GenericUDAFStd());
@@ -430,6 +428,8 @@ public final class FunctionRegistry {
registerGenericUDAF("ewah_bitmap", new GenericUDAFEWAHBitmap());
+ registerGenericUDAF("compute_stats" , new GenericUDAFComputeStats());
+
registerUDAF("percentile", UDAFPercentile.class);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java Wed Nov 7 04:55:00 2012
@@ -81,6 +81,9 @@ public abstract class Task<T extends Ser
// Descendants tasks who subscribe feeds from this task
protected transient List<Task<? extends Serializable>> feedSubscribers;
+ protected String id;
+ protected T work;
+
public static enum FeedType {
DYNAMIC_PARTITIONS, // list of dynamic partitions
};
@@ -324,8 +327,7 @@ public abstract class Task<T extends Ser
return isrunnable;
}
- protected String id;
- protected T work;
+
public void setWork(T work) {
this.work = work;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java Wed Nov 7 04:55:00 2012
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask;
import org.apache.hadoop.hive.ql.io.rcfile.merge.MergeWork;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
import org.apache.hadoop.hive.ql.plan.ConditionalWork;
import org.apache.hadoop.hive.ql.plan.CopyWork;
import org.apache.hadoop.hive.ql.plan.DDLWork;
@@ -77,6 +78,7 @@ public final class TaskFactory {
MapredLocalTask.class));
taskvec.add(new taskTuple<StatsWork>(StatsWork.class,
StatsTask.class));
+ taskvec.add(new taskTuple<ColumnStatsWork>(ColumnStatsWork.class, ColumnStatsTask.class));
taskvec.add(new taskTuple<MergeWork>(MergeWork.class,
BlockMergeTask.class));
taskvec.add(new taskTuple<DependencyCollectionWork>(DependencyCollectionWork.class,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Wed Nov 7 04:55:00 2012
@@ -41,7 +41,6 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsShell;
@@ -56,6 +55,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.Constants;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -2185,6 +2185,65 @@ public class Hive {
return indexes;
}
+ public boolean updateTableColumnStatistics(ColumnStatistics statsObj) throws HiveException {
+ try {
+ return getMSC().updateTableColumnStatistics(statsObj);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+ }
+
+ public boolean updatePartitionColumnStatistics(ColumnStatistics statsObj) throws HiveException {
+ try {
+ return getMSC().updatePartitionColumnStatistics(statsObj);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+ }
+
+ public ColumnStatistics getTableColumnStatistics(String dbName, String tableName, String colName)
+ throws HiveException {
+ try {
+ return getMSC().getTableColumnStatistics(dbName, tableName, colName);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+
+ }
+
+ public ColumnStatistics getPartitionColumnStatistics(String dbName, String tableName,
+ String partName, String colName) throws HiveException {
+ try {
+ return getMSC().getPartitionColumnStatistics(dbName, tableName, partName, colName);
+ } catch (Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+ }
+
+ public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName)
+ throws HiveException {
+ try {
+ return getMSC().deleteTableColumnStatistics(dbName, tableName, colName);
+ } catch(Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+ }
+
+ public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName,
+ String colName) throws HiveException {
+ try {
+ return getMSC().deletePartitionColumnStatistics(dbName, tableName, partName, colName);
+ } catch(Exception e) {
+ LOG.error(StringUtils.stringifyException(e));
+ throw new HiveException(e);
+ }
+ }
+
public Table newTable(String tableName) throws HiveException {
String[] names = getQualifiedNames(tableName);
switch (names.length) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java Wed Nov 7 04:55:00 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.parse.S
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+
/**
* RewriteParseContextGenerator is a class that offers methods to generate operator tree
* for input queries. It is implemented on lines of the analyzeInternal(..) method
@@ -108,7 +109,7 @@ public final class RewriteParseContextGe
ASTNode child = ast;
ParseContext subPCtx = ((SemanticAnalyzer) sem).getParseContext();
subPCtx.setContext(ctx);
- ((SemanticAnalyzer) sem).init(subPCtx);
+ ((SemanticAnalyzer) sem).initParseCtx(subPCtx);
LOG.info("Starting Sub-query Semantic Analysis");
sem.doPhase1(child, qb, sem.initPhase1Ctx());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Wed Nov 7 04:55:00 2012
@@ -242,9 +242,17 @@ public abstract class BaseSemanticAnalyz
}
public abstract void analyzeInternal(ASTNode ast) throws SemanticException;
+ public void init() {
+ //no-op
+ }
- public void analyze(ASTNode ast, Context ctx) throws SemanticException {
+ public void initCtx(Context ctx) {
this.ctx = ctx;
+ }
+
+ public void analyze(ASTNode ast, Context ctx) throws SemanticException {
+ initCtx(ctx);
+ init();
analyzeInternal(ast);
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1406465&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Wed Nov 7 04:55:00 2012
@@ -0,0 +1,548 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * ColumnStatsSemanticAnalyzer.
+ * Handles semantic analysis and rewrite for gathering column statistics both at the level of a
+ * partition and a table. Note that table statistics are implemented in SemanticAnalyzer.
+ *
+ */
+public class ColumnStatsSemanticAnalyzer extends SemanticAnalyzer {
+ private static final Log LOG = LogFactory
+ .getLog(ColumnStatsSemanticAnalyzer.class);
+
+ private ASTNode originalTree;
+ private ASTNode rewrittenTree;
+ private String rewrittenQuery;
+
+ private Context ctx;
+ private boolean isRewritten;
+
+ private boolean isTableLevel;
+ private String tableName;
+ private List<String> colNames;
+ private List<String> colType;
+ private String partName;
+
+ private class PartitionList {
+ private final String[] partKeys;
+ private String[] partKeyTypes;
+ private final String[] partValues;
+ private int numPartitions;
+ private int numPartitionValues;
+
+ PartitionList(int numPartitions) {
+ this.numPartitions = numPartitions;
+ partKeys = new String[numPartitions];
+ partValues = new String[numPartitions];
+ }
+
+ public int getNumPartitions() {
+ return numPartitions;
+ }
+
+ public void setNumPartitions(int numPartitions) {
+ this.numPartitions = numPartitions;
+ }
+
+ public String[] getPartValues() {
+ return partValues;
+ }
+
+ public String[] getPartKeys() {
+ return partKeys;
+ }
+
+ public void addPartValue(String partValue, int index) {
+ partValues[index] = new String(partValue);
+ }
+
+ public void addPartKey(String partKey, int index) {
+ partKeys[index] = new String(partKey);
+ }
+
+ public int getNumPartValues() {
+ return numPartitionValues;
+ }
+
+ public void setNumPartValues(int numPartValues) {
+ numPartitionValues = numPartValues;
+ }
+
+ public String[] getPartKeyTypes() {
+ return partKeyTypes;
+ }
+
+ public void setPartKeyTypes(String[] partKeyTypes) {
+ this.partKeyTypes = partKeyTypes;
+ }
+
+ public void setPartKeyType(String partKeyType, int index) {
+ partKeyTypes[index] = partKeyType;
+ }
+ }
+
+ public ColumnStatsSemanticAnalyzer(HiveConf conf) throws SemanticException {
+ super(conf);
+ }
+
+ private boolean shouldRewrite(ASTNode tree) {
+ boolean rwt = false;
+ if (tree.getChildCount() > 1) {
+ ASTNode child0 = (ASTNode) tree.getChild(0);
+ ASTNode child1;
+ if (child0.getToken().getType() == HiveParser.TOK_TAB) {
+ child0 = (ASTNode) child0.getChild(0);
+ if (child0.getToken().getType() == HiveParser.TOK_TABNAME) {
+ child1 = (ASTNode) tree.getChild(1);
+ if (child1.getToken().getType() == HiveParser.TOK_TABCOLNAME) {
+ rwt = true;
+ }
+ }
+ }
+ }
+ return rwt;
+ }
+
+ private boolean isPartitionLevelStats(ASTNode tree) {
+ boolean isPartitioned = false;
+ ASTNode child = (ASTNode) tree.getChild(0);
+ if (child.getChildCount() > 1) {
+ child = (ASTNode) child.getChild(1);
+ if (child.getToken().getType() == HiveParser.TOK_PARTSPEC) {
+ isPartitioned = true;
+ }
+ }
+ return isPartitioned;
+ }
+
+ private String getTableName(ASTNode tree) {
+ return getUnescapedName((ASTNode) tree.getChild(0).getChild(0));
+ }
+
+ private PartitionList getPartKeyValuePairsFromAST(ASTNode tree) {
+ ASTNode child = ((ASTNode) tree.getChild(0).getChild(1));
+ int numParts = child.getChildCount();
+ PartitionList partList = new PartitionList(numParts);
+ String partKey;
+ String partValue;
+ int numPartValue = 0;
+ for (int i = 0; i < numParts; i++) {
+ partKey = new String(getUnescapedName((ASTNode) child.getChild(i).getChild(0)));
+ if (child.getChild(i).getChildCount() > 1) {
+ partValue = new String(getUnescapedName((ASTNode) child.getChild(i).getChild(1)));
+ partValue = partValue.replaceAll("'", "");
+ numPartValue += 1;
+ } else {
+ partValue = null;
+ }
+ partList.addPartKey(partKey, i);
+ if (partValue != null) {
+ partList.addPartValue(partValue, i);
+ }
+ }
+ partList.setNumPartValues(numPartValue);
+ return partList;
+ }
+
+ private List<String> getColumnName(ASTNode tree) {
+ int numCols = tree.getChild(1).getChildCount();
+ List<String> colName = new LinkedList<String>();
+ for (int i = 0; i < numCols; i++) {
+ colName.add(i, new String(getUnescapedName((ASTNode) tree.getChild(1).getChild(i))));
+ }
+ return colName;
+ }
+
+ private int getNumColumns(ASTNode tree) {
+ return tree.getChild(1).getChildCount();
+ }
+
+ private void validatePartitionKeys(String tableName, PartitionList partList) throws
+ SemanticException {
+ Table tbl;
+ try {
+ tbl = db.getTable(tableName);
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
+ }
+
+ List<FieldSchema> partKeys = tbl.getPartitionKeys();
+ String[] inputPartKeys = partList.getPartKeys();
+
+ if (inputPartKeys.length != partKeys.size()) {
+ throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INCORRECT_NUM_PART_KEY.getMsg());
+ }
+
+ Map<String, Integer> partKeysMap = new LinkedHashMap<String, Integer>();
+ for (int i=0; i <inputPartKeys.length; i++) {
+ partKeysMap.put(inputPartKeys[i].toLowerCase(), new Integer(1));
+ }
+ // Verify that the user specified part keys match the part keys in the table
+ for (FieldSchema partKey:partKeys) {
+ if (!partKeysMap.containsKey(partKey.getName().toLowerCase())) {
+ throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PART_KEY.getMsg());
+ }
+ }
+ }
+
+ private String[] getPartitionKeysType(String tableName, PartitionList partList) throws
+ SemanticException {
+ Table tbl;
+ try {
+ tbl = db.getTable(tableName);
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
+ }
+
+ List<FieldSchema> partKeys = tbl.getPartitionKeys();
+ String[] inputPartKeys = partList.getPartKeys();
+ String[] inputPartKeyTypes = new String[inputPartKeys.length];
+
+ for (int i=0; i < inputPartKeys.length; i++) {
+ for (FieldSchema partKey:partKeys) {
+ if (inputPartKeys[i].equalsIgnoreCase(partKey.getName())) {
+ inputPartKeyTypes[i] = new String(partKey.getType());
+ break;
+ }
+ }
+ }
+ return inputPartKeyTypes;
+ }
+
+ private String constructPartitionName(String tableName, PartitionList partList)
+ throws SemanticException {
+ Table tbl;
+ Partition part;
+ String[] partKeys = partList.getPartKeys();
+ String[] partValues = partList.getPartValues();
+
+ try {
+ tbl = db.getTable(tableName);
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
+ }
+
+ Map<String, String> partSpec = new LinkedHashMap<String, String>();
+ for (int i=0; i<partKeys.length; i++) {
+ partSpec.put(partKeys[i].toLowerCase(), partValues[i]);
+ }
+
+ try {
+ part = db.getPartition(tbl, partSpec, false);
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partName));
+ }
+
+ if (part == null) {
+ throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_PARTITION.getMsg());
+ }
+ return part.getName();
+ }
+
+ private void validatePartitionClause(String tableName, PartitionList partList) throws
+ SemanticException {
+ int numPartKeys = partList.getNumPartitions();
+ int numPartValues = partList.getNumPartValues();
+ // Raise error if the user has specified dynamic partitions in the partitioning clause
+ if (numPartKeys != numPartValues) {
+ throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_INVALID_SYNTAX.getMsg());
+ }
+ // Validate the user specified partition keys match the partition keys in the table
+ validatePartitionKeys(tableName, partList);
+ }
+
+ private StringBuilder genPartitionClause(PartitionList partList) throws SemanticException {
+ StringBuilder whereClause = new StringBuilder(" where ");
+ boolean predPresent = false;
+ StringBuilder groupByClause = new StringBuilder(" group by ");
+ boolean aggPresent = false;
+ StringBuilder retClause = null;
+ String[] partKeys = partList.getPartKeys();
+ String[] partValues = partList.getPartValues();
+ String[] partKeysType = getPartitionKeysType(tableName, partList);
+
+ for (int i = 0; i < partList.getNumPartitions(); i++) {
+ if (partValues[i] != null) {
+ if (!predPresent) {
+ whereClause.append(partKeys[i]);
+ whereClause.append(" = ");
+ if (partKeysType[i].equalsIgnoreCase("string")) {
+ whereClause.append("'");
+ }
+ whereClause.append(partValues[i]);
+ if (partKeysType[i].equalsIgnoreCase("string")) {
+ whereClause.append("'");
+ }
+ predPresent = true;
+ } else {
+ whereClause.append(" and ");
+ whereClause.append(partKeys[i]);
+ whereClause.append(" = ");
+ if (partKeysType[i].equalsIgnoreCase("string")) {
+ whereClause.append("'");
+ }
+ whereClause.append(partValues[i]);
+ if (partKeysType[i].equalsIgnoreCase("string")) {
+ whereClause.append("'");
+ }
+ }
+ } else {
+ if (!aggPresent) {
+ groupByClause.append(partKeys[i]);
+ aggPresent = true;
+ } else {
+ groupByClause.append(",");
+ groupByClause.append(partKeys[i]);
+ }
+ }
+ }
+ // attach the predicate and group by to the return clause
+ if (predPresent) {
+ retClause = new StringBuilder(whereClause);
+ }
+ if (aggPresent) {
+ retClause.append(groupByClause);
+ }
+ return retClause;
+ }
+
+ private int getNumBitVectorsForNDVEstimation(HiveConf conf) {
+ int numBitVectors;
+ float percentageError = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVE_STATS_NDV_ERROR);
+
+ if (percentageError <= 2.4) {
+ numBitVectors = 1024;
+ } else if (percentageError <= 3.4 ) {
+ numBitVectors = 1024;
+ } else if (percentageError <= 4.8) {
+ numBitVectors = 512;
+ } else if (percentageError <= 6.8) {
+ numBitVectors = 256;
+ } else if (percentageError <= 9.7) {
+ numBitVectors = 128;
+ } else if (percentageError <= 13.8) {
+ numBitVectors = 64;
+ } else if (percentageError <= 19.6) {
+ numBitVectors = 32;
+ } else if (percentageError <= 28.2) {
+ numBitVectors = 16;
+ } else if (percentageError <= 40.9) {
+ numBitVectors = 8;
+ } else if (percentageError <= 61.0) {
+ numBitVectors = 4;
+ } else {
+ numBitVectors = 2;
+ }
+ return numBitVectors;
+ }
+
+ private List<String> getTableColumnType(String tableName, List<String> colNames, int numCols)
+ throws SemanticException{
+ List<String> colTypes = new LinkedList<String>();
+ String colName;
+ Table tbl;
+ try {
+ tbl = db.getTable(tableName);
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
+ }
+
+ List<FieldSchema> cols = tbl.getCols();
+
+ for (int i=0; i <numCols; i++) {
+ colName = colNames.get(i);
+ for (FieldSchema col: cols) {
+ if (colName.equalsIgnoreCase(col.getName())) {
+ colTypes.add(i, new String(col.getType()));
+ }
+ }
+ }
+ return colTypes;
+ }
+
+ private List<String> getPartitionColumnType(String tableName, String partName,
+ List<String> colNames, int numCols) throws SemanticException {
+ List<String> colTypes = new LinkedList<String>();
+ String colName;
+ Table tbl;
+ try {
+ tbl = db.getTable(tableName);
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
+ }
+
+ List<String> partNames = new ArrayList<String>();
+ partNames.add(partName);
+ List<Partition> partitionList;
+
+ try {
+ partitionList = db.getPartitionsByNames(tbl, partNames);
+ } catch (HiveException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(partName));
+ }
+ Partition part = partitionList.get(0);
+ List<FieldSchema> cols = part.getCols();
+
+ for (int i=0; i <numCols; i++) {
+ colName = colNames.get(i);
+ for (FieldSchema col: cols) {
+ if (colName.equalsIgnoreCase(col.getName())) {
+ colTypes.add(i, new String(col.getType()));
+ }
+ }
+ }
+ return colTypes;
+ }
+
+ private String genRewrittenQuery(List<String> colNames, int numBitVectors, PartitionList partList,
+ boolean isPartitionStats) throws SemanticException{
+ StringBuilder rewrittenQueryBuilder = new StringBuilder("select ");
+ String rewrittenQuery;
+ for (int i = 0; i < colNames.size(); i++) {
+ if (i > 0) {
+ rewrittenQueryBuilder.append(" , ");
+ }
+ rewrittenQueryBuilder.append("compute_stats(");
+ rewrittenQueryBuilder.append(colNames.get(i));
+ rewrittenQueryBuilder.append(" , ");
+ rewrittenQueryBuilder.append(numBitVectors);
+ rewrittenQueryBuilder.append(" )");
+ }
+ rewrittenQueryBuilder.append(" from ");
+ rewrittenQueryBuilder.append(tableName);
+ isRewritten = true;
+
+ // If partition level statistics is requested, add predicate and group by as needed to rewritten
+ // query
+ if (isPartitionStats) {
+ rewrittenQueryBuilder.append(genPartitionClause(partList));
+ }
+
+ rewrittenQuery = rewrittenQueryBuilder.toString();
+ rewrittenQuery = new VariableSubstitution().substitute(conf, rewrittenQuery);
+ return rewrittenQuery;
+ }
+
+ private ASTNode genRewrittenTree(String rewrittenQuery) throws SemanticException {
+ ASTNode rewrittenTree;
+ // Parse the rewritten query string
+ try {
+ ctx = new Context(conf);
+ } catch (IOException e) {
+ throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_IO_ERROR.getMsg());
+ }
+ ctx.setCmd(rewrittenQuery);
+ ParseDriver pd = new ParseDriver();
+
+ try {
+ rewrittenTree = pd.parse(rewrittenQuery, ctx);
+ } catch (ParseException e) {
+ throw new SemanticException(ErrorMsg.COLUMNSTATSCOLLECTOR_PARSE_ERROR.getMsg());
+ }
+ rewrittenTree = ParseUtils.findRootNonNullToken(rewrittenTree);
+ return rewrittenTree;
+ }
+
+ public ColumnStatsSemanticAnalyzer(HiveConf conf, ASTNode tree) throws SemanticException {
+ super(conf);
+ /* Rewrite only analyze table <> column <> compute statistics; Don't rewrite analyze table
+ * command - table stats are collected by the table scan operator and is not rewritten to
+ * an aggregation.
+ */
+ if (shouldRewrite(tree)) {
+ tableName = new String(getTableName(tree));
+ colNames = getColumnName(tree);
+ int numCols = getNumColumns(tree);
+ // Save away the original AST
+ originalTree = tree;
+ boolean isPartitionStats = isPartitionLevelStats(tree);
+ PartitionList partList = null;
+
+ if (isPartitionStats) {
+ isTableLevel = false;
+ partList = getPartKeyValuePairsFromAST(tree);
+ validatePartitionClause(tableName, partList);
+ partName = constructPartitionName(tableName, partList);
+ colType = getPartitionColumnType(tableName, partName, colNames, numCols);
+ } else {
+ isTableLevel = true;
+ colType = getTableColumnType(tableName, colNames, numCols);
+ }
+
+ int numBitVectors = getNumBitVectorsForNDVEstimation(conf);
+ rewrittenQuery = genRewrittenQuery(colNames, numBitVectors, partList, isPartitionStats);
+ rewrittenTree = genRewrittenTree(rewrittenQuery);
+ } else {
+ // Not an analyze table column compute statistics statement - don't do any rewrites
+ originalTree = rewrittenTree = tree;
+ rewrittenQuery = null;
+ isRewritten = false;
+ }
+ }
+
+ @Override
+ public void analyze(ASTNode ast, Context origCtx) throws SemanticException {
+ QB qb;
+ QBParseInfo qbp;
+
+ // initialize QB
+ init();
+
+ // Setup the necessary metadata if originating from analyze rewrite
+ if (isRewritten) {
+ qb = getQB();
+ qb.setAnalyzeRewrite(true);
+ qbp = qb.getParseInfo();
+ qbp.setTableName(tableName);
+ qbp.setTblLvl(isTableLevel);
+
+ if (!isTableLevel) {
+ qbp.setPartName(partName);
+ }
+ qbp.setColName(colNames);
+ qbp.setColType(colType);
+ initCtx(ctx);
+ LOG.info("Invoking analyze on rewritten query");
+ analyzeInternal(rewrittenTree);
+ } else {
+ initCtx(origCtx);
+ LOG.info("Invoking analyze on original query");
+ analyzeInternal(originalTree);
+ }
+ }
+}
Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Wed Nov 7 04:55:00 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.ql.parse;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_CASCADE;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASECOMMENT;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASELOCATION;
+import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DATABASEPROPERTIES;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_IFEXISTS;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_IFNOTEXISTS;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_SHOWDATABASES;
@@ -669,10 +671,10 @@ public class DDLSemanticAnalyzer extends
case TOK_DATABASECOMMENT:
dbComment = unescapeSQLString(childNode.getChild(0).getText());
break;
- case HiveParser.TOK_DATABASEPROPERTIES:
+ case TOK_DATABASEPROPERTIES:
dbProps = DDLSemanticAnalyzer.getProps((ASTNode) childNode.getChild(0));
break;
- case HiveParser.TOK_DATABASELOCATION:
+ case TOK_DATABASELOCATION:
dbLocation = unescapeSQLString(childNode.getChild(0).getText());
break;
default:
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java Wed Nov 7 04:55:00 2012
@@ -20,14 +20,10 @@ package org.apache.hadoop.hive.ql.parse;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.OutputStream;
import java.io.Serializable;
import java.net.URI;
import java.util.List;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.TransformerException;
-
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -38,15 +34,8 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.CopyWork;
-import org.apache.thrift.TException;
-import org.apache.thrift.TSerializer;
-import org.apache.thrift.protocol.TJSONProtocol;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
/**
* ExportSemanticAnalyzer.
@@ -133,5 +122,4 @@ public class ExportSemanticAnalyzer exte
outputs.add(new WriteEntity(toURI.toString(),
toURI.getScheme().equals("hdfs") ? true : false));
}
-
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java Wed Nov 7 04:55:00 2012
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hive.ql.parse;
-
-import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_IFEXISTS;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -64,7 +61,7 @@ public class FunctionSemanticAnalyzer ex
private void analyzeDropFunction(ASTNode ast) throws SemanticException {
String functionName = ast.getChild(0).getText();
- boolean ifExists = (ast.getFirstChildWithType(TOK_IFEXISTS) != null);
+ boolean ifExists = (ast.getFirstChildWithType(HiveParser.TOK_IFEXISTS) != null);
// we want to signal an error if the function doesn't exist and we're
// configured not to ignore this
boolean throwException =
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Wed Nov 7 04:55:00 2012
@@ -905,7 +905,7 @@ descStatement
analyzeStatement
@init { msgs.push("analyze statement"); }
@after { msgs.pop(); }
- : KW_ANALYZE KW_TABLE (parttype=tableOrPartition) KW_COMPUTE KW_STATISTICS -> ^(TOK_ANALYZE $parttype)
+ : KW_ANALYZE KW_TABLE (parttype=tableOrPartition) KW_COMPUTE KW_STATISTICS (KW_FOR KW_COLUMNS statsColumnName=columnNameList)? -> ^(TOK_ANALYZE $parttype $statsColumnName?)
;
showStatement
@@ -2508,6 +2508,7 @@ KW_CASCADE: 'CASCADE';
KW_SKEWED: 'SKEWED';
KW_ROLLUP: 'ROLLUP';
KW_CUBE: 'CUBE';
+KW_FOR: 'FOR';
// Operators
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Wed Nov 7 04:55:00 2012
@@ -539,5 +539,4 @@ public class ImportSemanticAnalyzer exte
}
return null;
}
-
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java Wed Nov 7 04:55:00 2012
@@ -49,6 +49,7 @@ public class QB {
private QBJoinTree qbjoin;
private String id;
private boolean isQuery;
+ private boolean isAnalyzeRewrite;
private CreateTableDesc tblDesc = null; // table descriptor of the final
// results
@@ -222,5 +223,14 @@ public class QB {
skewedColNames = tbl.getSkewedColNames();
}
return skewedColNames;
+
+ }
+
+ public boolean isAnalyzeRewrite() {
+ return isAnalyzeRewrite;
+ }
+
+ public void setAnalyzeRewrite(boolean isAnalyzeRewrite) {
+ this.isAnalyzeRewrite = isAnalyzeRewrite;
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java Wed Nov 7 04:55:00 2012
@@ -58,6 +58,12 @@ public class QBParseInfo {
private final HashMap<String, tableSpec> tableSpecs; // used for statistics
+ private String tableName; // used for column statistics
+ private List<String> colName; // used for column statistics
+ private List<String> colType; // used for column statistics
+ private String partName; // used for column statistics
+ private boolean isTblLvl; // used for column statistics
+
/**
* ClusterBy is a short name for both DistributeBy and SortBy.
*/
@@ -119,6 +125,7 @@ public class QBParseInfo {
aliasToLateralViews = new HashMap<String, ArrayList<ASTNode>>();
tableSpecs = new HashMap<String, BaseSemanticAnalyzer.tableSpec>();
+
}
public void setAggregationExprsForClause(String clause,
@@ -420,7 +427,7 @@ public class QBParseInfo {
while (iter.hasNext()) {
Map.Entry<String, ASTNode> entry = iter.next();
ASTNode v = entry.getValue();
- if (!(((ASTNode) v.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) {
+ if (!(((ASTNode)v.getChild(0)).getToken().getType() == HiveParser.TOK_TMP_FILE)) {
return false;
}
}
@@ -501,4 +508,52 @@ public class QBParseInfo {
public HashMap<String, TableSample> getNameToSample() {
return nameToSample;
}
+
+
+ protected static enum ClauseType {
+ CLUSTER_BY_CLAUSE,
+ DISTRIBUTE_BY_CLAUSE,
+ ORDER_BY_CLAUSE,
+ SORT_BY_CLAUSE
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public List<String> getColName() {
+ return colName;
+ }
+
+ public void setColName(List<String> colName) {
+ this.colName = colName;
+ }
+
+ public String getPartName() {
+ return partName;
+ }
+
+ public void setPartName(String partName) {
+ this.partName = partName;
+ }
+
+ public boolean isTblLvl() {
+ return isTblLvl;
+ }
+
+ public void setTblLvl(boolean isTblLvl) {
+ this.isTblLvl = isTblLvl;
+ }
+
+ public List<String> getColType() {
+ return colType;
+ }
+
+ public void setColType(List<String> colType) {
+ this.colType = colType;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Nov 7 04:55:00 2012
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.ql.QueryPr
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.ColumnStatsTask;
import org.apache.hadoop.hive.ql.exec.ConditionalTask;
import org.apache.hadoop.hive.ql.exec.ExecDriver;
import org.apache.hadoop.hive.ql.exec.FetchTask;
@@ -116,6 +117,8 @@ import org.apache.hadoop.hive.ql.optimiz
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
import org.apache.hadoop.hive.ql.plan.CreateViewDesc;
@@ -272,7 +275,7 @@ public class SemanticAnalyzer extends Ba
prunedPartitions.clear();
}
- public void init(ParseContext pctx) {
+ public void initParseCtx(ParseContext pctx) {
opToPartPruner = pctx.getOpToPartPruner();
opToPartList = pctx.getOpToPartList();
opToSamplePruner = pctx.getOpToSamplePruner();
@@ -829,7 +832,10 @@ public class SemanticAnalyzer extends Ba
case HiveParser.TOK_ANALYZE:
// Case of analyze command
+
String table_name = getUnescapedName((ASTNode)ast.getChild(0).getChild(0));
+
+
qb.setTabAlias(table_name, table_name);
qb.addAlias(table_name);
qb.getParseInfo().setIsAnalyzeCommand(true);
@@ -851,11 +857,13 @@ public class SemanticAnalyzer extends Ba
case HiveParser.TOK_INSERT:
ASTNode destination = (ASTNode) ast.getChild(0);
Tree tab = destination.getChild(0);
+
// Proceed if AST contains partition & If Not Exists
if (destination.getChildCount() == 2 &&
tab.getChildCount() == 2 &&
destination.getChild(1).getType() == HiveParser.TOK_IFNOTEXISTS) {
String tableName = tab.getChild(0).getChild(0).getText();
+
Tree partitions = tab.getChild(1);
int childCount = partitions.getChildCount();
HashMap<String, String> partition = new HashMap<String, String>();
@@ -2004,8 +2012,9 @@ public class SemanticAnalyzer extends Ba
private List<Integer> getGroupingSetsForCube(int size) {
int count = 1 << size;
List<Integer> results = new ArrayList<Integer>(count);
- for(int i = 0; i < count; ++i)
+ for(int i = 0; i < count; ++i) {
results.add(i);
+ }
return results;
}
@@ -7522,19 +7531,61 @@ public class SemanticAnalyzer extends Ba
}
}
+ /**
+ * A helper function to generate a column stats task on top of map-red task. The column stats
+ * task fetches from the output of the map-red task, constructs the column stats object and
+ * persists it to the metastore.
+ *
+ * This method generates a plan with a column stats task on top of map-red task and sets up the
+ * appropriate metadata to be used during execution.
+ *
+ * @param qb
+ */
+ private void genColumnStatsTask(QB qb) {
+ QBParseInfo qbParseInfo = qb.getParseInfo();
+ ColumnStatsTask cStatsTask = null;
+ ColumnStatsWork cStatsWork = null;
+ FetchWork fetch = null;
+ String tableName = qbParseInfo.getTableName();
+ String partName = qbParseInfo.getPartName();
+ List<String> colName = qbParseInfo.getColName();
+ List<String> colType = qbParseInfo.getColType();
+ boolean isTblLevel = qbParseInfo.isTblLvl();
+
+ String cols = loadFileWork.get(0).getColumns();
+ String colTypes = loadFileWork.get(0).getColumnTypes();
+
+ String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
+ TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
+
+ fetch = new FetchWork(new Path(loadFileWork.get(0).getSourceDir()).toString(),
+ resultTab, qb.getParseInfo().getOuterQueryLimit());
+
+ ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tableName, partName,
+ colName, colType, isTblLevel);
+ cStatsWork = new ColumnStatsWork(fetch, cStatsDesc);
+ cStatsTask = (ColumnStatsTask) TaskFactory.get(cStatsWork, conf);
+ rootTasks.add(cStatsTask);
+ }
+
@SuppressWarnings("nls")
private void genMapRedTasks(ParseContext pCtx) throws SemanticException {
+ boolean isCStats = qb.isAnalyzeRewrite();
+
if (pCtx.getFetchTask() != null) {
// replaced by single fetch task
- init(pCtx);
+ initParseCtx(pCtx);
return;
}
- init(pCtx);
+ initParseCtx(pCtx);
List<Task<MoveWork>> mvTask = new ArrayList<Task<MoveWork>>();
- // In case of a select, use a fetch task instead of a move task
- if (qb.getIsQuery()) {
+ /* In case of a select, use a fetch task instead of a move task.
+ * If the select is from analyze table column rewrite, don't create a fetch task. Instead create
+ * a column stats task later.
+ */
+ if (qb.getIsQuery() && !isCStats) {
if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) {
throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg());
}
@@ -7559,8 +7610,7 @@ public class SemanticAnalyzer extends Ba
+ ". Doesn't qualify limit optimiztion.");
globalLimitCtx.disableOpt();
}
-
- } else {
+ } else if (!isCStats) {
for (LoadTableDesc ltd : loadTableWork) {
Task<MoveWork> tsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
conf);
@@ -7579,7 +7629,6 @@ public class SemanticAnalyzer extends Ba
}
}
-
boolean oneLoadFile = true;
for (LoadFileDesc lfd : loadFileWork) {
if (qb.isCTAS()) {
@@ -7678,6 +7727,13 @@ public class SemanticAnalyzer extends Ba
topNodes.addAll(topOps.values());
ogw.startWalking(topNodes, null);
+ /* If the query was the result of analyze table column compute statistics rewrite, create
+ * a column stats task instead of a fetch task to persist stats to the metastore.
+ */
+ if (isCStats) {
+ genColumnStatsTask(qb);
+ }
+
// reduce sink does not have any kids - since the plan by now has been
// broken up into multiple
// tasks, iterate over all tasks.
@@ -7955,14 +8011,20 @@ public class SemanticAnalyzer extends Ba
}
@Override
- @SuppressWarnings("nls")
- public void analyzeInternal(ASTNode ast) throws SemanticException {
+ public void init() {
+ // clear most members
reset();
+ // init
QB qb = new QB(null, null, false);
this.qb = qb;
- this.ast = ast;
+ }
+
+ @Override
+ @SuppressWarnings("nls")
+ public void analyzeInternal(ASTNode ast) throws SemanticException {
ASTNode child = ast;
+ this.ast = ast;
viewsExpanded = new ArrayList<String>();
LOG.info("Starting Semantic Analysis");
@@ -8005,6 +8067,7 @@ public class SemanticAnalyzer extends Ba
// such as JDBC would prefer instead of the c0, c1 we'll end
// up with later.
Operator sinkOp = genPlan(qb);
+
resultSchema =
convertRowSchemaToViewSchema(opParseCtx.get(sinkOp).getRowResolver());
@@ -8895,4 +8958,12 @@ public class SemanticAnalyzer extends Ba
return conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS);
}
+
+ public QB getQB() {
+ return qb;
+ }
+
+ public void setQB(QB qb) {
+ this.qb = qb;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Wed Nov 7 04:55:00 2012
@@ -92,6 +92,7 @@ public final class SemanticAnalyzerFacto
commandType.put(HiveParser.TOK_ALTERDATABASE_PROPERTIES, HiveOperation.ALTERDATABASE);
commandType.put(HiveParser.TOK_DESCDATABASE, HiveOperation.DESCDATABASE);
commandType.put(HiveParser.TOK_ALTERTABLE_SKEWED, HiveOperation.ALTERTABLE_SKEWED);
+ commandType.put(HiveParser.TOK_ANALYZE, HiveOperation.ANALYZE_TABLE);
}
static {
@@ -200,9 +201,14 @@ public final class SemanticAnalyzerFacto
}
setSessionCommandType(commandType);
return new DDLSemanticAnalyzer(conf);
+
case HiveParser.TOK_CREATEFUNCTION:
case HiveParser.TOK_DROPFUNCTION:
return new FunctionSemanticAnalyzer(conf);
+
+ case HiveParser.TOK_ANALYZE:
+ return new ColumnStatsSemanticAnalyzer(conf, tree);
+
default:
return new SemanticAnalyzer(conf);
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java?rev=1406465&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java Wed Nov 7 04:55:00 2012
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.plan;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Contains the information needed to persist column level statistics
+ */
+public class ColumnStatsDesc extends DDLDesc implements Serializable, Cloneable {
+
+ private static final long serialVersionUID = 1L;
+ private boolean isTblLevel;
+ private String tableName;
+ private String partName;
+ private List<String> colName;
+ private List<String> colType;
+
+ public ColumnStatsDesc() {
+ }
+
+ public ColumnStatsDesc(String tableName, String partName, List<String> colName,
+ List<String> colType, boolean isTblLevel) {
+ this.tableName = tableName;
+ this.colName = colName;
+ this.colType = colType;
+ this.partName = partName;
+ this.isTblLevel = isTblLevel;
+ }
+
+ @Explain(displayName = "Table")
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ @Explain(displayName = "Is Table Level Stats", normalExplain=false)
+ public boolean isTblLevel() {
+ return isTblLevel;
+ }
+
+ public void setTblLevel(boolean isTblLevel) {
+ this.isTblLevel = isTblLevel;
+ }
+
+ @Explain(displayName = "Partition")
+ public String getPartName() {
+ return partName;
+ }
+
+ public void setPartName(String partName) {
+ this.partName = partName;
+ }
+
+ @Explain(displayName = "Columns")
+ public List<String> getColName() {
+ return colName;
+ }
+
+ public void setColName(List<String> colName) {
+ this.colName = colName;
+ }
+
+ @Explain(displayName = "Column Types")
+ public List<String> getColType() {
+ return colType;
+ }
+
+ public void setColType(List<String> colType) {
+ this.colType = colType;
+ }
+}
Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDesc.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java?rev=1406465&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java Wed Nov 7 04:55:00 2012
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.plan;
+
+import java.io.Serializable;
+
+/**
+ * ColumnStats Work.
+ *
+ */
+@Explain(displayName = "Column Stats Work")
+public class ColumnStatsWork implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private FetchWork fWork;
+ private ColumnStatsDesc colStats;
+
+ public ColumnStatsWork() {
+ }
+
+ public ColumnStatsWork(FetchWork work, ColumnStatsDesc colStats) {
+ this.fWork = work;
+ this.setColStats(colStats);
+ }
+
+ @Override
+ public String toString() {
+ String ret;
+ ret = fWork.toString();
+ return ret;
+ }
+
+ public FetchWork getfWork() {
+ return fWork;
+ }
+
+ public void setfWork(FetchWork fWork) {
+ this.fWork = fWork;
+ }
+
+ @Explain(displayName = "Column Stats Desc")
+ public ColumnStatsDesc getColStats() {
+ return colStats;
+ }
+
+ public void setColStats(ColumnStatsDesc colStats) {
+ this.colStats = colStats;
+ }
+}
Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsWork.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java?rev=1406465&r1=1406464&r2=1406465&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java Wed Nov 7 04:55:00 2012
@@ -49,6 +49,7 @@ public enum HiveOperation {
ALTERTABLE_SERDEPROPERTIES("ALTERTABLE_SERDEPROPERTIES", new Privilege[]{Privilege.ALTER_METADATA}, null),
ALTERPARTITION_SERDEPROPERTIES("ALTERPARTITION_SERDEPROPERTIES", new Privilege[]{Privilege.ALTER_METADATA}, null),
ALTERTABLE_CLUSTER_SORT("ALTERTABLE_CLUSTER_SORT", new Privilege[]{Privilege.ALTER_METADATA}, null),
+ ANALYZE_TABLE("ANALYZE_TABLE", null, null),
SHOWDATABASES("SHOWDATABASES", new Privilege[]{Privilege.SHOW_DATABASE}, null),
SHOWTABLES("SHOWTABLES", null, null),
SHOWCOLUMNS("SHOWCOLUMNS", null, null),
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/DoubleNumDistinctValueEstimator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/DoubleNumDistinctValueEstimator.java?rev=1406465&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/DoubleNumDistinctValueEstimator.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/DoubleNumDistinctValueEstimator.java Wed Nov 7 04:55:00 2012
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+public class DoubleNumDistinctValueEstimator extends NumDistinctValueEstimator {
+
+ public DoubleNumDistinctValueEstimator(int numBitVectors) {
+ super(numBitVectors);
+ }
+
+ public DoubleNumDistinctValueEstimator(String s, int numVectors) {
+ super(s, numVectors);
+ }
+
+ public void addToEstimator(double d) {
+ int v = new Double(d).hashCode();
+ super.addToEstimator(v);
+ }
+
+ public void addToEstimatorPCSA(double d) {
+ int v = new Double(d).hashCode();
+ super.addToEstimatorPCSA(v);
+ }
+}
Propchange: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/DoubleNumDistinctValueEstimator.java
------------------------------------------------------------------------------
svn:eol-style = native