You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2018/02/14 10:00:36 UTC
[12/15] hive git commit: HIVE-18448: Drop Support For Indexes From
Apache Hive (Zoltan Haindrich reviewed by Ashutosh Chauhan)
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
deleted file mode 100644
index 658422c..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteQueryUsingAggregateIndexCtx.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer.index;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.hadoop.hive.ql.optimizer.FieldNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.OperatorUtils;
-import org.apache.hadoop.hive.ql.exec.RowSchema;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.optimizer.ColumnPrunerProcFactory;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AggregationDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.GroupByDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-
-/**
- * RewriteQueryUsingAggregateIndexCtx class stores the
- * context for the {@link RewriteQueryUsingAggregateIndex}
- * used to rewrite operator plan with index table instead of base table.
- */
-
-public final class RewriteQueryUsingAggregateIndexCtx implements NodeProcessorCtx {
- private static final Logger LOG = LoggerFactory.getLogger(RewriteQueryUsingAggregateIndexCtx.class.getName());
- private RewriteQueryUsingAggregateIndexCtx(ParseContext parseContext, Hive hiveDb,
- RewriteCanApplyCtx canApplyCtx) {
- this.parseContext = parseContext;
- this.hiveDb = hiveDb;
- this.canApplyCtx = canApplyCtx;
- this.indexTableName = canApplyCtx.getIndexTableName();
- this.alias = canApplyCtx.getAlias();
- this.aggregateFunction = canApplyCtx.getAggFunction();
- this.indexKey = canApplyCtx.getIndexKey();
- }
-
- public static RewriteQueryUsingAggregateIndexCtx getInstance(ParseContext parseContext,
- Hive hiveDb, RewriteCanApplyCtx canApplyCtx) {
- return new RewriteQueryUsingAggregateIndexCtx(
- parseContext, hiveDb, canApplyCtx);
- }
-
- // Assumes one instance of this + single-threaded compilation for each query.
- private final Hive hiveDb;
- private final ParseContext parseContext;
- private final RewriteCanApplyCtx canApplyCtx;
- //We need the GenericUDAFEvaluator for GenericUDAF function "sum"
- private GenericUDAFEvaluator eval = null;
- private final String indexTableName;
- private final String alias;
- private final String aggregateFunction;
- private ExprNodeColumnDesc aggrExprNode = null;
- private String indexKey;
-
- public ParseContext getParseContext() {
- return parseContext;
- }
-
- public Hive getHiveDb() {
- return hiveDb;
- }
-
- public String getIndexName() {
- return indexTableName;
- }
-
- public GenericUDAFEvaluator getEval() {
- return eval;
- }
-
- public void setEval(GenericUDAFEvaluator eval) {
- this.eval = eval;
- }
-
- public void setAggrExprNode(ExprNodeColumnDesc aggrExprNode) {
- this.aggrExprNode = aggrExprNode;
- }
-
- public ExprNodeColumnDesc getAggrExprNode() {
- return aggrExprNode;
- }
-
- public String getAlias() {
- return alias;
- }
-
- public String getAggregateFunction() {
- return aggregateFunction;
- }
-
- public String getIndexKey() {
- return indexKey;
- }
-
- public void setIndexKey(String indexKey) {
- this.indexKey = indexKey;
- }
-
- public void invokeRewriteQueryProc() throws SemanticException {
- this.replaceTableScanProcess(canApplyCtx.getTableScanOperator());
- //We need aggrExprNode. Thus, replaceGroupByOperatorProcess should come before replaceSelectOperatorProcess
- for (int index = 0; index < canApplyCtx.getGroupByOperators().size(); index++) {
- this.replaceGroupByOperatorProcess(canApplyCtx.getGroupByOperators().get(index), index);
- }
- for (SelectOperator selectperator : canApplyCtx.getSelectOperators()) {
- this.replaceSelectOperatorProcess(selectperator);
- }
- }
-
- /**
- * This method replaces the original TableScanOperator with the new
- * TableScanOperator and metadata that scans over the index table rather than
- * scanning over the original table.
- *
- */
- private void replaceTableScanProcess(TableScanOperator scanOperator) throws SemanticException {
- RewriteQueryUsingAggregateIndexCtx rewriteQueryCtx = this;
- String alias = rewriteQueryCtx.getAlias();
-
- // Need to remove the original TableScanOperators from these data structures
- // and add new ones
- HashMap<String, TableScanOperator> topOps = rewriteQueryCtx.getParseContext()
- .getTopOps();
-
- // remove original TableScanOperator
- topOps.remove(alias);
-
- String indexTableName = rewriteQueryCtx.getIndexName();
- Table indexTableHandle = null;
- try {
- indexTableHandle = rewriteQueryCtx.getHiveDb().getTable(indexTableName);
- } catch (HiveException e) {
- LOG.error("Error while getting the table handle for index table.");
- LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
- throw new SemanticException(e.getMessage(), e);
- }
-
- // construct a new descriptor for the index table scan
- TableScanDesc indexTableScanDesc = new TableScanDesc(indexTableHandle);
- indexTableScanDesc.setGatherStats(false);
-
- String k = org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.encodeTableName(indexTableName) + Path.SEPARATOR;
- indexTableScanDesc.setStatsAggPrefix(k);
- scanOperator.setConf(indexTableScanDesc);
-
- // Construct the new RowResolver for the new TableScanOperator
- ArrayList<ColumnInfo> sigRS = new ArrayList<ColumnInfo>();
- try {
- StructObjectInspector rowObjectInspector = (StructObjectInspector) indexTableHandle
- .getDeserializer().getObjectInspector();
- StructField field = rowObjectInspector.getStructFieldRef(rewriteQueryCtx.getIndexKey());
- sigRS.add(new ColumnInfo(field.getFieldName(), TypeInfoUtils.getTypeInfoFromObjectInspector(
- field.getFieldObjectInspector()), indexTableName, false));
- } catch (SerDeException e) {
- LOG.error("Error while creating the RowResolver for new TableScanOperator.");
- LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
- throw new SemanticException(e.getMessage(), e);
- }
- RowSchema rs = new RowSchema(sigRS);
-
- // Set row resolver for new table
- String newAlias = indexTableName;
- int index = alias.lastIndexOf(":");
- if (index >= 0) {
- newAlias = alias.substring(0, index) + ":" + indexTableName;
- }
-
- // Scan operator now points to other table
- scanOperator.getConf().setAlias(newAlias);
- scanOperator.setAlias(indexTableName);
- topOps.put(newAlias, scanOperator);
- rewriteQueryCtx.getParseContext().setTopOps(topOps);
-
- ColumnPrunerProcFactory.setupNeededColumns(scanOperator, rs,
- Arrays.asList(new FieldNode(rewriteQueryCtx.getIndexKey())));
- }
-
- /**
- * This method replaces the original SelectOperator with the new
- * SelectOperator with a new column indexed_key_column.
- */
- private void replaceSelectOperatorProcess(SelectOperator operator) throws SemanticException {
- RewriteQueryUsingAggregateIndexCtx rewriteQueryCtx = this;
- // we need to set the colList, outputColumnNames, colExprMap,
- // rowSchema for only that SelectOperator which precedes the GroupByOperator
- // count(indexed_key_column) needs to be replaced by
- // sum(`_count_of_indexed_key_column`)
- List<ExprNodeDesc> selColList = operator.getConf().getColList();
- selColList.add(rewriteQueryCtx.getAggrExprNode());
-
- List<String> selOutputColNames = operator.getConf().getOutputColumnNames();
- selOutputColNames.add(rewriteQueryCtx.getAggrExprNode().getColumn());
-
- operator.getColumnExprMap().put(rewriteQueryCtx.getAggrExprNode().getColumn(),
- rewriteQueryCtx.getAggrExprNode());
-
- RowSchema selRS = operator.getSchema();
- List<ColumnInfo> selRSSignature = selRS.getSignature();
- // Need to create a new type for Column[_count_of_indexed_key_column] node
- PrimitiveTypeInfo pti = TypeInfoFactory.getPrimitiveTypeInfo("bigint");
- pti.setTypeName("bigint");
- ColumnInfo newCI = new ColumnInfo(rewriteQueryCtx.getAggregateFunction(), pti, "", false);
- selRSSignature.add(newCI);
- selRS.setSignature((ArrayList<ColumnInfo>) selRSSignature);
- operator.setSchema(selRS);
- }
-
- /**
- * We need to replace the count(indexed_column_key) GenericUDAF aggregation
- * function for group-by construct to "sum" GenericUDAF. This method creates a
- * new operator tree for a sample query that creates a GroupByOperator with
- * sum aggregation function and uses that GroupByOperator information to
- * replace the original GroupByOperator aggregation information. It replaces
- * the AggregationDesc (aggregation descriptor) of the old GroupByOperator
- * with the new Aggregation Desc of the new GroupByOperator.
- * @return
- */
- private void replaceGroupByOperatorProcess(GroupByOperator operator, int index)
- throws SemanticException {
- RewriteQueryUsingAggregateIndexCtx rewriteQueryCtx = this;
-
- // We need to replace the GroupByOperator which is before RS
- if (index == 0) {
- // the query contains the sum aggregation GenericUDAF
- String selReplacementCommand = "select sum(`" + rewriteQueryCtx.getAggregateFunction() + "`)"
- + " from `" + rewriteQueryCtx.getIndexName() + "` group by "
- + rewriteQueryCtx.getIndexKey() + " ";
- // retrieve the operator tree for the query, and the required GroupByOperator from it
- Operator<?> newOperatorTree = RewriteParseContextGenerator.generateOperatorTree(
- rewriteQueryCtx.getParseContext().getQueryState(),
- selReplacementCommand);
-
- // we get our new GroupByOperator here
- GroupByOperator newGbyOperator = OperatorUtils.findLastOperatorUpstream(
- newOperatorTree, GroupByOperator.class);
- if (newGbyOperator == null) {
- throw new SemanticException("Error replacing GroupBy operator.");
- }
-
- // we need this information to set the correct colList, outputColumnNames
- // in SelectOperator
- ExprNodeColumnDesc aggrExprNode = null;
-
- // Construct the new AggregationDesc to get rid of the current
- // internal names and replace them with new internal names
- // as required by the operator tree
- GroupByDesc newConf = newGbyOperator.getConf();
- List<AggregationDesc> newAggrList = newConf.getAggregators();
- if (newAggrList != null && newAggrList.size() > 0) {
- for (AggregationDesc aggregationDesc : newAggrList) {
- rewriteQueryCtx.setEval(aggregationDesc.getGenericUDAFEvaluator());
- aggrExprNode = (ExprNodeColumnDesc) aggregationDesc.getParameters().get(0);
- rewriteQueryCtx.setAggrExprNode(aggrExprNode);
- }
- }
-
- // Now the GroupByOperator has the new AggregationList;
- // sum(`_count_of_indexed_key`)
- // instead of count(indexed_key)
- GroupByDesc oldConf = operator.getConf();
- oldConf.setAggregators((ArrayList<AggregationDesc>) newAggrList);
- operator.setConf(oldConf);
-
- } else {
- // we just need to reset the GenericUDAFEvaluator and its name for this
- // GroupByOperator whose parent is the ReduceSinkOperator
- GroupByDesc childConf = operator.getConf();
- List<AggregationDesc> childAggrList = childConf.getAggregators();
- if (childAggrList != null && childAggrList.size() > 0) {
- for (AggregationDesc aggregationDesc : childAggrList) {
- List<ExprNodeDesc> paraList = aggregationDesc.getParameters();
- List<ObjectInspector> parametersOIList = new ArrayList<ObjectInspector>();
- for (ExprNodeDesc expr : paraList) {
- parametersOIList.add(expr.getWritableObjectInspector());
- }
- GenericUDAFEvaluator evaluator = FunctionRegistry.getGenericUDAFEvaluator("sum",
- parametersOIList, false, false);
- aggregationDesc.setGenericUDAFEvaluator(evaluator);
- aggregationDesc.setGenericUDAFName("sum");
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java
deleted file mode 100644
index d204fe8..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer.physical;
-
-import java.util.ArrayList;
-
-import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
-import org.apache.hadoop.hive.ql.lib.Dispatcher;
-import org.apache.hadoop.hive.ql.lib.GraphWalker;
-import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.optimizer.physical.index.IndexWhereTaskDispatcher;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-
-public class IndexWhereResolver implements PhysicalPlanResolver {
-
- @Override
- public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticException {
- Dispatcher dispatcher = new IndexWhereTaskDispatcher(physicalContext);
- GraphWalker opGraphWalker = new DefaultGraphWalker(dispatcher);
- ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.addAll(physicalContext.getRootTasks());
- opGraphWalker.startWalking(topNodes, null);
-
- return physicalContext;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
index a64a498..0f3c5f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
@@ -60,9 +60,6 @@ public class PhysicalOptimizer {
}
}
- if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEOPTINDEXFILTER)) {
- resolvers.add(new IndexWhereResolver());
- }
resolvers.add(new MapJoinResolver());
if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {
resolvers.add(new MetadataOnlyOptimizer());
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java
deleted file mode 100644
index 179d4c2..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer.physical.index;
-
-import java.io.Serializable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
-
-public class IndexWhereProcCtx implements NodeProcessorCtx {
-
- private static final Logger LOG = LoggerFactory.getLogger(IndexWhereProcCtx.class.getName());
-
- private final Task<? extends Serializable> currentTask;
- private final ParseContext parseCtx;
-
- public IndexWhereProcCtx(Task<? extends Serializable> task, ParseContext parseCtx) {
- this.currentTask = task;
- this.parseCtx = parseCtx;
- }
-
- public ParseContext getParseContext() {
- return parseCtx;
- }
-
- public Task<? extends Serializable> getCurrentTask() {
- return currentTask;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java
deleted file mode 100644
index b284afa..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer.physical.index;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Stack;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.ContentSummary;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
-import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
-import org.apache.hadoop.hive.ql.index.HiveIndexQueryContext;
-import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.HiveUtils;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.optimizer.IndexUtils;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.MapWork;
-import org.apache.hadoop.hive.ql.plan.MapredWork;
-import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-
-/**
-*
-* IndexWhereProcessor.
-* Processes Operator Nodes to look for WHERE queries with a predicate column
-* on which we have an index. Creates an index subquery Task for these
-* WHERE queries to use the index automatically.
-*/
-public class IndexWhereProcessor implements NodeProcessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(IndexWhereProcessor.class.getName());
- private final Map<TableScanOperator, List<Index>> tsToIndices;
-
- public IndexWhereProcessor(Map<TableScanOperator, List<Index>> tsToIndices) {
- super();
- this.tsToIndices = tsToIndices;
- }
-
- @Override
- /**
- * Process a node of the operator tree. This matches on the rule in IndexWhereTaskDispatcher
- */
- public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
- Object... nodeOutputs) throws SemanticException {
-
- TableScanOperator operator = (TableScanOperator) nd;
- List<Node> opChildren = operator.getChildren();
- TableScanDesc operatorDesc = operator.getConf();
- if (operatorDesc == null || !tsToIndices.containsKey(operator)) {
- return null;
- }
- List<Index> indexes = tsToIndices.get(operator);
-
- ExprNodeDesc predicate = operatorDesc.getFilterExpr();
-
- IndexWhereProcCtx context = (IndexWhereProcCtx) procCtx;
- ParseContext pctx = context.getParseContext();
- LOG.info("Processing predicate for index optimization");
-
- if (predicate == null) {
- LOG.info("null predicate pushed down");
- return null;
- }
- LOG.info(predicate.getExprString());
-
- // check if we have tsToIndices on all partitions in this table scan
- Set<Partition> queryPartitions;
- try {
- queryPartitions = IndexUtils.checkPartitionsCoveredByIndex(operator, pctx, indexes);
- if (queryPartitions == null) { // partitions not covered
- return null;
- }
- } catch (HiveException e) {
- LOG.error("Fatal Error: problem accessing metastore", e);
- throw new SemanticException(e);
- }
-
- // we can only process MapReduce tasks to check input size
- if (!context.getCurrentTask().isMapRedTask()) {
- return null;
- }
- MapRedTask currentTask = (MapRedTask) context.getCurrentTask();
-
- // get potential reentrant index queries from each index
- Map<Index, HiveIndexQueryContext> queryContexts = new HashMap<Index, HiveIndexQueryContext>();
- // make sure we have an index on the table being scanned
- TableDesc tblDesc = operator.getTableDescSkewJoin();
-
- Map<String, List<Index>> indexesByType = new HashMap<String, List<Index>>();
- for (Index indexOnTable : indexes) {
- if (indexesByType.get(indexOnTable.getIndexHandlerClass()) == null) {
- List<Index> newType = new ArrayList<Index>();
- newType.add(indexOnTable);
- indexesByType.put(indexOnTable.getIndexHandlerClass(), newType);
- } else {
- indexesByType.get(indexOnTable.getIndexHandlerClass()).add(indexOnTable);
- }
- }
-
- // choose index type with most tsToIndices of the same type on the table
- // TODO HIVE-2130 This would be a good place for some sort of cost based choice?
- List<Index> bestIndexes = indexesByType.values().iterator().next();
- for (List<Index> indexTypes : indexesByType.values()) {
- if (bestIndexes.size() < indexTypes.size()) {
- bestIndexes = indexTypes;
- }
- }
-
- // rewrite index queries for the chosen index type
- HiveIndexQueryContext tmpQueryContext = new HiveIndexQueryContext();
- tmpQueryContext.setQueryPartitions(queryPartitions);
- rewriteForIndexes(predicate, bestIndexes, pctx, currentTask, tmpQueryContext);
- List<Task<?>> indexTasks = tmpQueryContext.getQueryTasks();
-
- if (indexTasks != null && indexTasks.size() > 0) {
- queryContexts.put(bestIndexes.get(0), tmpQueryContext);
- }
- // choose an index rewrite to use
- if (queryContexts.size() > 0) {
- // TODO HIVE-2130 This would be a good place for some sort of cost based choice?
- Index chosenIndex = queryContexts.keySet().iterator().next();
-
- // modify the parse context to use indexing
- // we need to delay this until we choose one index so that we don't attempt to modify pctx multiple times
- HiveIndexQueryContext queryContext = queryContexts.get(chosenIndex);
-
- // prepare the map reduce job to use indexing
- MapWork work = currentTask.getWork().getMapWork();
- work.setInputformat(queryContext.getIndexInputFormat());
- work.addIndexIntermediateFile(queryContext.getIndexIntermediateFile());
- // modify inputs based on index query
- Set<ReadEntity> inputs = pctx.getSemanticInputs();
- inputs.addAll(queryContext.getAdditionalSemanticInputs());
- List<Task<?>> chosenRewrite = queryContext.getQueryTasks();
-
- // add dependencies so index query runs first
- insertIndexQuery(pctx, context, chosenRewrite);
- }
-
- return null;
- }
-
- /**
- * Get a list of Tasks to activate use of tsToIndices.
- * Generate the tasks for the index query (where we store results of
- * querying the index in a tmp file) inside the IndexHandler
- * @param predicate Predicate of query to rewrite
- * @param index Index to use for rewrite
- * @param pctx
- * @param task original task before rewrite
- * @param queryContext stores return values
- */
- private void rewriteForIndexes(ExprNodeDesc predicate, List<Index> indexes,
- ParseContext pctx, Task<MapredWork> task,
- HiveIndexQueryContext queryContext)
- throws SemanticException {
- HiveIndexHandler indexHandler;
- // All tsToIndices in the list are of the same type, and therefore can use the
- // same handler to generate the index query tasks
- Index index = indexes.get(0);
- try {
- indexHandler = HiveUtils.getIndexHandler(pctx.getConf(), index.getIndexHandlerClass());
- } catch (HiveException e) {
- LOG.error("Exception while loading IndexHandler: " + index.getIndexHandlerClass(), e);
- throw new SemanticException("Failed to load indexHandler: " + index.getIndexHandlerClass(), e);
- }
-
- // check the size
- try {
- ContentSummary inputSummary = Utilities.getInputSummary(pctx.getContext(), task.getWork().getMapWork(), null);
- long inputSize = inputSummary.getLength();
- if (!indexHandler.checkQuerySize(inputSize, pctx.getConf())) {
- queryContext.setQueryTasks(null);
- return;
- }
- } catch (IOException e) {
- throw new SemanticException("Failed to get task size", e);
- }
-
- // use the IndexHandler to generate the index query
- indexHandler.generateIndexQuery(indexes, predicate, pctx, queryContext);
- // TODO HIVE-2115 use queryContext.residualPredicate to process residual predicate
-
- return;
- }
-
-
- /**
- * Insert the rewrite tasks at the head of the pctx task tree
- * @param pctx
- * @param context
- * @param chosenRewrite
- */
- private void insertIndexQuery(ParseContext pctx, IndexWhereProcCtx context, List<Task<?>> chosenRewrite) {
- Task<?> wholeTableScan = context.getCurrentTask();
- LinkedHashSet<Task<?>> rewriteLeaves = new LinkedHashSet<Task<?>>();
- findLeaves(chosenRewrite, rewriteLeaves);
-
- for (Task<?> leaf : rewriteLeaves) {
- leaf.addDependentTask(wholeTableScan); // add full scan task as child for every index query task
- }
-
- // replace the original with the index sub-query as a root task
- pctx.replaceRootTask(wholeTableScan, chosenRewrite);
- }
-
- /**
- * Find the leaves of the task tree
- */
- private void findLeaves(List<Task<?>> tasks, Set<Task<?>> leaves) {
- for (Task<?> t : tasks) {
- if (t.getDependentTasks() == null) {
- leaves.add(t);
- } else {
- findLeaves(t.getDependentTasks(), leaves);
- }
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
deleted file mode 100644
index c9dae8f..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.optimizer.physical.index;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Stack;
-
-import com.google.common.collect.Maps;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.metastore.cache.CacheUtils;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler;
-import org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler;
-import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
-import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
-import org.apache.hadoop.hive.ql.lib.Dispatcher;
-import org.apache.hadoop.hive.ql.lib.GraphWalker;
-import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
-import org.apache.hadoop.hive.ql.lib.Rule;
-import org.apache.hadoop.hive.ql.lib.RuleRegExp;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.optimizer.IndexUtils;
-import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.MapredWork;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hive.common.util.HiveStringUtils;
-
-/**
- *
- * IndexWhereTaskDispatcher. Walks a Task tree, and for the right kind of Task,
- * walks the operator tree to create an index subquery. Then attaches the
- * subquery task to the task tree.
- *
- */
-public class IndexWhereTaskDispatcher implements Dispatcher {
-
- private final PhysicalContext physicalContext;
- // To store table to index mapping
- private final Map<String, List<Index>> indexMap;
- private final List<String> supportedIndexes;
-
- public IndexWhereTaskDispatcher(PhysicalContext context) {
- super();
- physicalContext = context;
- indexMap = Maps.newHashMap();
- supportedIndexes = new ArrayList<String>();
- supportedIndexes.add(CompactIndexHandler.class.getName());
- supportedIndexes.add(BitmapIndexHandler.class.getName());
- }
-
- @Override
- public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs)
- throws SemanticException {
-
- Task<? extends Serializable> task = (Task<? extends Serializable>) nd;
-
- ParseContext pctx = physicalContext.getParseContext();
-
- // create the regex's so the walker can recognize our WHERE queries
- Map<Rule, NodeProcessor> operatorRules = createOperatorRules(pctx);
-
- // check for no indexes on any table
- if (operatorRules == null) {
- return null;
- }
-
- // create context so the walker can carry the current task with it.
- IndexWhereProcCtx indexWhereOptimizeCtx = new IndexWhereProcCtx(task, pctx);
-
- // create the dispatcher, which fires the processor according to the rule that
- // best matches
- Dispatcher dispatcher = new DefaultRuleDispatcher(getDefaultProcessor(),
- operatorRules,
- indexWhereOptimizeCtx);
-
- // walk the mapper operator(not task) tree for each specific task
- GraphWalker ogw = new DefaultGraphWalker(dispatcher);
- ArrayList<Node> topNodes = new ArrayList<Node>();
- if (task.getWork() instanceof MapredWork) {
- topNodes.addAll(((MapredWork)task.getWork()).getMapWork().getAliasToWork().values());
- } else {
- return null;
- }
- ogw.startWalking(topNodes, null);
-
- return null;
- }
-
- private List<Index> getIndex(Table table) throws SemanticException {
- String indexCacheKey = CacheUtils.buildKey(
- HiveStringUtils.normalizeIdentifier(table.getDbName()),
- HiveStringUtils.normalizeIdentifier(table.getTableName()));
- List<Index>indexList = indexMap.get(indexCacheKey);
- if (indexList == null) {
- indexList = IndexUtils.getIndexes(table, supportedIndexes);
- if (indexList == null) {
- indexList = Collections.emptyList();
- }
- indexMap.put(indexCacheKey, indexList);
- }
- return indexList;
- }
-
- /**
- * Create a set of rules that only matches WHERE predicates on columns we have
- * an index on.
- * @return
- */
- private Map<Rule, NodeProcessor> createOperatorRules(ParseContext pctx) throws SemanticException {
- Map<Rule, NodeProcessor> operatorRules = new LinkedHashMap<Rule, NodeProcessor>();
-
- // query the metastore to know what columns we have indexed
- Map<TableScanOperator, List<Index>> indexes = new HashMap<TableScanOperator, List<Index>>();
- for (Operator<? extends OperatorDesc> op : pctx.getTopOps().values()) {
- if (op instanceof TableScanOperator) {
- List<Index> tblIndexes = getIndex(((TableScanOperator) op).getConf().getTableMetadata());
- if (tblIndexes.size() > 0) {
- indexes.put((TableScanOperator) op, tblIndexes);
- }
- }
- }
-
- // quit if our tables don't have any indexes
- if (indexes.size() == 0) {
- return null;
- }
-
- // We set the pushed predicate from the WHERE clause as the filter expr on
- // all table scan operators, so we look for table scan operators(TS%)
- operatorRules.put(new RuleRegExp("RULEWhere", TableScanOperator.getOperatorName() + "%"),
- new IndexWhereProcessor(indexes));
-
- return operatorRules;
- }
-
-
- private NodeProcessor getDefaultProcessor() {
- return new NodeProcessor() {
- @Override
- public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
- Object... nodeOutputs) throws SemanticException {
- return null;
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 4338fa6..41d878f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -43,7 +43,6 @@ import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -52,9 +51,7 @@ import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
@@ -82,9 +79,6 @@ import org.apache.hadoop.hive.ql.hooks.Entity.Type;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity.WriteType;
-import org.apache.hadoop.hive.ql.index.HiveIndex;
-import org.apache.hadoop.hive.ql.index.HiveIndex.IndexType;
-import org.apache.hadoop.hive.ql.index.HiveIndexHandler;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
@@ -106,8 +100,6 @@ import org.apache.hadoop.hive.ql.plan.AbortTxnsDesc;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
import org.apache.hadoop.hive.ql.plan.AddPartitionDesc.OnePartitionDesc;
import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.AlterIndexDesc;
-import org.apache.hadoop.hive.ql.plan.AlterIndexDesc.AlterIndexTypes;
import org.apache.hadoop.hive.ql.plan.AlterMaterializedViewDesc;
import org.apache.hadoop.hive.ql.plan.AlterMaterializedViewDesc.AlterMaterializedViewTypes;
import org.apache.hadoop.hive.ql.plan.AlterResourcePlanDesc;
@@ -121,7 +113,6 @@ import org.apache.hadoop.hive.ql.plan.BasicStatsWork;
import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc;
import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork;
import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.CreateIndexDesc;
import org.apache.hadoop.hive.ql.plan.CreateOrAlterWMMappingDesc;
import org.apache.hadoop.hive.ql.plan.CreateOrAlterWMPoolDesc;
import org.apache.hadoop.hive.ql.plan.CreateOrDropTriggerToPoolMappingDesc;
@@ -132,7 +123,6 @@ import org.apache.hadoop.hive.ql.plan.DescDatabaseDesc;
import org.apache.hadoop.hive.ql.plan.DescFunctionDesc;
import org.apache.hadoop.hive.ql.plan.DescTableDesc;
import org.apache.hadoop.hive.ql.plan.DropDatabaseDesc;
-import org.apache.hadoop.hive.ql.plan.DropIndexDesc;
import org.apache.hadoop.hive.ql.plan.DropResourcePlanDesc;
import org.apache.hadoop.hive.ql.plan.DropTableDesc;
import org.apache.hadoop.hive.ql.plan.DropWMMappingDesc;
@@ -162,7 +152,6 @@ import org.apache.hadoop.hive.ql.plan.ShowCreateTableDesc;
import org.apache.hadoop.hive.ql.plan.ShowDatabasesDesc;
import org.apache.hadoop.hive.ql.plan.ShowFunctionsDesc;
import org.apache.hadoop.hive.ql.plan.ShowGrantDesc;
-import org.apache.hadoop.hive.ql.plan.ShowIndexesDesc;
import org.apache.hadoop.hive.ql.plan.ShowLocksDesc;
import org.apache.hadoop.hive.ql.plan.ShowPartitionsDesc;
import org.apache.hadoop.hive.ql.plan.ShowResourcePlanDesc;
@@ -176,7 +165,6 @@ import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.TruncateTableDesc;
import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
-import org.apache.hadoop.hive.ql.session.LineageState;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -373,12 +361,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
case HiveParser.TOK_TRUNCATETABLE:
analyzeTruncateTable(ast);
break;
- case HiveParser.TOK_CREATEINDEX:
- analyzeCreateIndex(ast);
- break;
- case HiveParser.TOK_DROPINDEX:
- analyzeDropIndex(ast);
- break;
case HiveParser.TOK_DESCTABLE:
ctx.setResFile(ctx.getLocalTmpPath());
analyzeDescribeTable(ast);
@@ -485,12 +467,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
}
break;
}
- case HiveParser.TOK_ALTERINDEX_REBUILD:
- analyzeAlterIndexRebuild(ast);
- break;
- case HiveParser.TOK_ALTERINDEX_PROPERTIES:
- analyzeAlterIndexProps(ast);
- break;
case HiveParser.TOK_SHOWPARTITIONS:
ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowPartitions(ast);
@@ -503,10 +479,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
ctx.setResFile(ctx.getLocalTmpPath());
analyzeShowCreateTable(ast);
break;
- case HiveParser.TOK_SHOWINDEXES:
- ctx.setResFile(ctx.getLocalTmpPath());
- analyzeShowIndexes(ast);
- break;
case HiveParser.TOK_LOCKTABLE:
analyzeLockTable(ast);
break;
@@ -1486,11 +1458,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
try {
columnNames = getColumnNames((ASTNode)ast.getChild(1));
- // Throw an error if the table is indexed
- List<Index> indexes = db.getIndexes(table.getDbName(), tableName, (short)1);
- if (indexes != null && indexes.size() > 0) {
- throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_INDEXED_TABLE.getMsg());
- }
// It would be possible to support this, but this is such a pointless command.
if (AcidUtils.isInsertOnlyTable(table.getParameters())) {
throw new SemanticException("Truncating MM table columns not presently supported");
@@ -1634,235 +1601,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
return true;
}
- private void analyzeCreateIndex(ASTNode ast) throws SemanticException {
- String indexName = unescapeIdentifier(ast.getChild(0).getText());
- String typeName = unescapeSQLString(ast.getChild(1).getText());
- String[] qTabName = getQualifiedTableName((ASTNode) ast.getChild(2));
- List<String> indexedCols = getColumnNames((ASTNode) ast.getChild(3));
-
- IndexType indexType = HiveIndex.getIndexType(typeName);
- if (indexType != null) {
- typeName = indexType.getHandlerClsName();
- } else {
- try {
- JavaUtils.loadClass(typeName);
- } catch (Exception e) {
- throw new SemanticException("class name provided for index handler not found.", e);
- }
- }
-
- String indexTableName = null;
- boolean deferredRebuild = false;
- String location = null;
- Map<String, String> tblProps = null;
- Map<String, String> idxProps = null;
- String indexComment = null;
-
- RowFormatParams rowFormatParams = new RowFormatParams();
- StorageFormat storageFormat = new StorageFormat(conf);
-
- for (int idx = 4; idx < ast.getChildCount(); idx++) {
- ASTNode child = (ASTNode) ast.getChild(idx);
- if (storageFormat.fillStorageFormat(child)) {
- continue;
- }
- switch (child.getToken().getType()) {
- case HiveParser.TOK_TABLEROWFORMAT:
- rowFormatParams.analyzeRowFormat(child);
- break;
- case HiveParser.TOK_CREATEINDEX_INDEXTBLNAME:
- ASTNode ch = (ASTNode) child.getChild(0);
- indexTableName = getUnescapedName(ch);
- break;
- case HiveParser.TOK_DEFERRED_REBUILDINDEX:
- deferredRebuild = true;
- break;
- case HiveParser.TOK_TABLELOCATION:
- location = unescapeSQLString(child.getChild(0).getText());
- addLocationToOutputs(location);
- break;
- case HiveParser.TOK_TABLEPROPERTIES:
- tblProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0));
- break;
- case HiveParser.TOK_INDEXPROPERTIES:
- idxProps = DDLSemanticAnalyzer.getProps((ASTNode) child.getChild(0));
- break;
- case HiveParser.TOK_TABLESERIALIZER:
- child = (ASTNode) child.getChild(0);
- storageFormat.setSerde(unescapeSQLString(child.getChild(0).getText()));
- if (child.getChildCount() == 2) {
- readProps((ASTNode) (child.getChild(1).getChild(0)),
- storageFormat.getSerdeProps());
- }
- break;
- case HiveParser.TOK_INDEXCOMMENT:
- child = (ASTNode) child.getChild(0);
- indexComment = unescapeSQLString(child.getText());
- }
- }
-
- storageFormat.fillDefaultStorageFormat(false, false);
- if (indexTableName == null) {
- indexTableName = MetaStoreUtils.getIndexTableName(qTabName[0], qTabName[1], indexName);
- indexTableName = qTabName[0] + "." + indexTableName; // on same database with base table
- } else {
- indexTableName = getDotName(Utilities.getDbTableName(indexTableName));
- }
- inputs.add(new ReadEntity(getTable(qTabName)));
-
- CreateIndexDesc crtIndexDesc = new CreateIndexDesc(getDotName(qTabName), indexName,
- indexedCols, indexTableName, deferredRebuild, storageFormat.getInputFormat(),
- storageFormat.getOutputFormat(),
- storageFormat.getStorageHandler(), typeName, location, idxProps, tblProps,
- storageFormat.getSerde(), storageFormat.getSerdeProps(), rowFormatParams.collItemDelim,
- rowFormatParams.fieldDelim, rowFormatParams.fieldEscape,
- rowFormatParams.lineDelim, rowFormatParams.mapKeyDelim, indexComment);
- Task<?> createIndex =
- TaskFactory.get(new DDLWork(getInputs(), getOutputs(), crtIndexDesc), conf);
- rootTasks.add(createIndex);
- }
-
- private void analyzeDropIndex(ASTNode ast) throws SemanticException {
- String indexName = unescapeIdentifier(ast.getChild(0).getText());
- String tableName = getUnescapedName((ASTNode) ast.getChild(1));
- boolean ifExists = (ast.getFirstChildWithType(HiveParser.TOK_IFEXISTS) != null);
- // we want to signal an error if the index doesn't exist and we're
- // configured not to ignore this
- boolean throwException =
- !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT);
- Table tbl = getTable(tableName, false);
- if (throwException && tbl == null) {
- throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tableName));
- }
- try {
- Index idx = db.getIndex(tableName, indexName);
- } catch (HiveException e) {
- if (!(e.getCause() instanceof NoSuchObjectException)) {
- throw new SemanticException(ErrorMsg.CANNOT_DROP_INDEX.getMsg("dropping index"), e);
- }
- if (throwException) {
- throw new SemanticException(ErrorMsg.INVALID_INDEX.getMsg(indexName));
- }
- }
- if (tbl != null) {
- inputs.add(new ReadEntity(tbl));
- }
-
- DropIndexDesc dropIdxDesc = new DropIndexDesc(indexName, tableName, throwException);
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- dropIdxDesc), conf));
- }
-
- private void analyzeAlterIndexRebuild(ASTNode ast) throws SemanticException {
- String[] qualified = getQualifiedTableName((ASTNode) ast.getChild(0));
- String indexName = unescapeIdentifier(ast.getChild(1).getText());
- HashMap<String, String> partSpec = null;
- Tree part = ast.getChild(2);
- if (part != null) {
- partSpec = getValidatedPartSpec(getTable(qualified), (ASTNode)part, conf, false);
- }
- List<Task<?>> indexBuilder = getIndexBuilderMapRed(qualified, indexName, partSpec);
- rootTasks.addAll(indexBuilder);
-
- // Handle updating index timestamps
- AlterIndexDesc alterIdxDesc = new AlterIndexDesc(AlterIndexTypes.UPDATETIMESTAMP);
- alterIdxDesc.setIndexName(indexName);
- alterIdxDesc.setBaseTableName(getDotName(qualified));
- alterIdxDesc.setSpec(partSpec);
-
- Task<?> tsTask = TaskFactory.get(new DDLWork(alterIdxDesc), conf);
- for (Task<?> t : indexBuilder) {
- t.addDependentTask(tsTask);
- }
- }
-
- private void analyzeAlterIndexProps(ASTNode ast)
- throws SemanticException {
-
- String[] qualified = getQualifiedTableName((ASTNode) ast.getChild(0));
- String indexName = unescapeIdentifier(ast.getChild(1).getText());
- HashMap<String, String> mapProp = getProps((ASTNode) (ast.getChild(2))
- .getChild(0));
-
- AlterIndexDesc alterIdxDesc = new AlterIndexDesc(AlterIndexTypes.ADDPROPS);
- alterIdxDesc.setProps(mapProp);
- alterIdxDesc.setIndexName(indexName);
- alterIdxDesc.setBaseTableName(getDotName(qualified));
-
- rootTasks.add(TaskFactory.get(new DDLWork(alterIdxDesc), conf));
- }
-
- private List<Task<?>> getIndexBuilderMapRed(String[] names, String indexName,
- HashMap<String, String> partSpec) throws SemanticException {
- try {
- Index index = db.getIndex(names[0], names[1], indexName);
- Table indexTbl = null;
- String indexTableName = index.getIndexTableName();
- if (indexTableName != null) {
- indexTbl = getTable(Utilities.getDbTableName(index.getDbName(), indexTableName));
- }
- Table baseTbl = getTable(new String[] {index.getDbName(), index.getOrigTableName()});
-
- String handlerCls = index.getIndexHandlerClass();
- HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls);
-
- List<Partition> indexTblPartitions = null;
- List<Partition> baseTblPartitions = null;
- if (indexTbl != null) {
- indexTblPartitions = new ArrayList<Partition>();
- baseTblPartitions = preparePartitions(baseTbl, partSpec,
- indexTbl, db, indexTblPartitions);
- }
-
- LineageState lineageState = queryState.getLineageState();
- List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl,
- index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs(),
- lineageState);
- return ret;
- } catch (Exception e) {
- throw new SemanticException(e);
- }
- }
-
- private List<Partition> preparePartitions(
- org.apache.hadoop.hive.ql.metadata.Table baseTbl,
- HashMap<String, String> partSpec,
- org.apache.hadoop.hive.ql.metadata.Table indexTbl, Hive db,
- List<Partition> indexTblPartitions)
- throws HiveException, MetaException {
- List<Partition> baseTblPartitions = new ArrayList<Partition>();
- if (partSpec != null) {
- // if partspec is specified, then only producing index for that
- // partition
- Partition part = db.getPartition(baseTbl, partSpec, false);
- if (part == null) {
- throw new HiveException("Partition "
- + Warehouse.makePartName(partSpec, false)
- + " does not exist in table "
- + baseTbl.getTableName());
- }
- baseTblPartitions.add(part);
- Partition indexPart = db.getPartition(indexTbl, partSpec, false);
- if (indexPart == null) {
- indexPart = db.createPartition(indexTbl, partSpec);
- }
- indexTblPartitions.add(indexPart);
- } else if (baseTbl.isPartitioned()) {
- // if no partition is specified, create indexes for all partitions one
- // by one.
- baseTblPartitions = db.getPartitions(baseTbl);
- for (Partition basePart : baseTblPartitions) {
- HashMap<String, String> pSpec = basePart.getSpec();
- Partition indexPart = db.getPartition(indexTbl, pSpec, false);
- if (indexPart == null) {
- indexPart = db.createPartition(indexTbl, pSpec);
- }
- indexTblPartitions.add(indexPart);
- }
- }
- return baseTblPartitions;
- }
-
private void validateAlterTableType(Table tbl, AlterTableTypes op) throws SemanticException {
validateAlterTableType(tbl, op, false);
}
@@ -2190,17 +1928,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
List<String> bucketCols = null;
Class<? extends InputFormat> inputFormatClass = null;
boolean isArchived = false;
- boolean checkIndex = HiveConf.getBoolVar(conf,
- HiveConf.ConfVars.HIVE_CONCATENATE_CHECK_INDEX);
- if (checkIndex) {
- List<Index> indexes = db.getIndexes(tblObj.getDbName(), tblObj.getTableName(),
- Short.MAX_VALUE);
- if (indexes != null && indexes.size() > 0) {
- throw new SemanticException("can not do merge because source table "
- + tableName + " is indexed.");
- }
- }
-
if (tblObj.isPartitioned()) {
if (partSpec == null) {
throw new SemanticException("source table " + tableName
@@ -2762,6 +2489,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
setFetchTask(createFetchTask(showCreateDbDesc.getSchema()));
}
+
private void analyzeShowCreateTable(ASTNode ast) throws SemanticException {
ShowCreateTableDesc showCreateTblDesc;
String tableName = getUnescapedName((ASTNode)ast.getChild(0));
@@ -2896,21 +2624,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
setFetchTask(createFetchTask(showTblPropertiesDesc.getSchema()));
}
- private void analyzeShowIndexes(ASTNode ast) throws SemanticException {
- ShowIndexesDesc showIndexesDesc;
- String tableName = getUnescapedName((ASTNode) ast.getChild(0));
- showIndexesDesc = new ShowIndexesDesc(tableName, ctx.getResFile());
-
- if (ast.getChildCount() == 2) {
- int descOptions = ast.getChild(1).getType();
- showIndexesDesc.setFormatted(descOptions == HiveParser.KW_FORMATTED);
- }
-
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- showIndexesDesc), conf));
- setFetchTask(createFetchTask(showIndexesDesc.getSchema()));
- }
-
/**
* Add the task according to the parsed command tree. This is used for the CLI
* command "SHOW FUNCTIONS;".
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 9073623..3e84fd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -151,10 +151,6 @@ TOK_COLTYPELIST;
TOK_CREATEDATABASE;
TOK_CREATETABLE;
TOK_TRUNCATETABLE;
-TOK_CREATEINDEX;
-TOK_CREATEINDEX_INDEXTBLNAME;
-TOK_DEFERRED_REBUILDINDEX;
-TOK_DROPINDEX;
TOK_LIKETABLE;
TOK_DESCTABLE;
TOK_DESCFUNCTION;
@@ -189,8 +185,6 @@ TOK_ALTERTABLE_CLUSTER_SORT;
TOK_ALTERTABLE_COMPACT;
TOK_ALTERTABLE_DROPCONSTRAINT;
TOK_ALTERTABLE_ADDCONSTRAINT;
-TOK_ALTERINDEX_REBUILD;
-TOK_ALTERINDEX_PROPERTIES;
TOK_MSCK;
TOK_SHOWDATABASES;
TOK_SHOWTABLES;
@@ -271,8 +265,6 @@ TOK_EXPLAIN_SQ_REWRITE;
TOK_TABLESERIALIZER;
TOK_TABLEPROPERTIES;
TOK_TABLEPROPLIST;
-TOK_INDEXPROPERTIES;
-TOK_INDEXPROPLIST;
TOK_TABTYPE;
TOK_LIMIT;
TOK_OFFSET;
@@ -310,7 +302,6 @@ TOK_PRIV_ALTER_METADATA;
TOK_PRIV_ALTER_DATA;
TOK_PRIV_DELETE;
TOK_PRIV_DROP;
-TOK_PRIV_INDEX;
TOK_PRIV_INSERT;
TOK_PRIV_LOCK;
TOK_PRIV_SELECT;
@@ -324,9 +315,7 @@ TOK_SHOW_ROLE_GRANT;
TOK_SHOW_ROLES;
TOK_SHOW_SET_ROLE;
TOK_SHOW_ROLE_PRINCIPALS;
-TOK_SHOWINDEXES;
TOK_SHOWDBLOCKS;
-TOK_INDEXCOMMENT;
TOK_DESCDATABASE;
TOK_DATABASEPROPERTIES;
TOK_DATABASELOCATION;
@@ -929,8 +918,6 @@ ddlStatement
| dropMaterializedViewStatement
| createFunctionStatement
| createMacroStatement
- | createIndexStatement
- | dropIndexStatement
| dropFunctionStatement
| reloadFunctionStatement
| dropMacroStatement
@@ -1102,80 +1089,6 @@ truncateTableStatement
@after { popMsg(state); }
: KW_TRUNCATE KW_TABLE tablePartitionPrefix (KW_COLUMNS LPAREN columnNameList RPAREN)? -> ^(TOK_TRUNCATETABLE tablePartitionPrefix columnNameList?);
-createIndexStatement
-@init { pushMsg("create index statement", state);}
-@after {popMsg(state);}
- : KW_CREATE KW_INDEX indexName=identifier
- KW_ON KW_TABLE tab=tableName LPAREN indexedCols=columnNameList RPAREN
- KW_AS typeName=StringLiteral
- autoRebuild?
- indexPropertiesPrefixed?
- indexTblName?
- tableRowFormat?
- tableFileFormat?
- tableLocation?
- tablePropertiesPrefixed?
- indexComment?
- ->^(TOK_CREATEINDEX $indexName $typeName $tab $indexedCols
- autoRebuild?
- indexPropertiesPrefixed?
- indexTblName?
- tableRowFormat?
- tableFileFormat?
- tableLocation?
- tablePropertiesPrefixed?
- indexComment?)
- ;
-
-indexComment
-@init { pushMsg("comment on an index", state);}
-@after {popMsg(state);}
- :
- KW_COMMENT comment=StringLiteral -> ^(TOK_INDEXCOMMENT $comment)
- ;
-
-autoRebuild
-@init { pushMsg("auto rebuild index", state);}
-@after {popMsg(state);}
- : KW_WITH KW_DEFERRED KW_REBUILD
- ->^(TOK_DEFERRED_REBUILDINDEX)
- ;
-
-indexTblName
-@init { pushMsg("index table name", state);}
-@after {popMsg(state);}
- : KW_IN KW_TABLE indexTbl=tableName
- ->^(TOK_CREATEINDEX_INDEXTBLNAME $indexTbl)
- ;
-
-indexPropertiesPrefixed
-@init { pushMsg("table properties with prefix", state); }
-@after { popMsg(state); }
- :
- KW_IDXPROPERTIES! indexProperties
- ;
-
-indexProperties
-@init { pushMsg("index properties", state); }
-@after { popMsg(state); }
- :
- LPAREN indexPropertiesList RPAREN -> ^(TOK_INDEXPROPERTIES indexPropertiesList)
- ;
-
-indexPropertiesList
-@init { pushMsg("index properties list", state); }
-@after { popMsg(state); }
- :
- keyValueProperty (COMMA keyValueProperty)* -> ^(TOK_INDEXPROPLIST keyValueProperty+)
- ;
-
-dropIndexStatement
-@init { pushMsg("drop index statement", state);}
-@after {popMsg(state);}
- : KW_DROP KW_INDEX ifExists? indexName=identifier KW_ON tab=tableName
- ->^(TOK_DROPINDEX $indexName $tab ifExists?)
- ;
-
dropTableStatement
@init { pushMsg("drop statement", state); }
@after { popMsg(state); }
@@ -1190,7 +1103,6 @@ alterStatement
| KW_ALTER KW_VIEW tableName KW_AS? alterViewStatementSuffix -> ^(TOK_ALTERVIEW tableName alterViewStatementSuffix)
| KW_ALTER KW_MATERIALIZED KW_VIEW tableName alterMaterializedViewStatementSuffix
-> ^(TOK_ALTER_MATERIALIZED_VIEW tableName alterMaterializedViewStatementSuffix)
- | KW_ALTER KW_INDEX alterIndexStatementSuffix -> alterIndexStatementSuffix
| KW_ALTER (KW_DATABASE|KW_SCHEMA) alterDatabaseStatementSuffix -> alterDatabaseStatementSuffix
;
@@ -1254,20 +1166,6 @@ alterMaterializedViewStatementSuffix
| alterMaterializedViewSuffixRebuild
;
-alterIndexStatementSuffix
-@init { pushMsg("alter index statement", state); }
-@after { popMsg(state); }
- : indexName=identifier KW_ON tableName partitionSpec?
- (
- KW_REBUILD
- ->^(TOK_ALTERINDEX_REBUILD tableName $indexName partitionSpec?)
- |
- KW_SET KW_IDXPROPERTIES
- indexProperties
- ->^(TOK_ALTERINDEX_PROPERTIES tableName $indexName indexProperties)
- )
- ;
-
alterDatabaseStatementSuffix
@init { pushMsg("alter database statement", state); }
@after { popMsg(state); }
@@ -1652,8 +1550,6 @@ showStatement
|
(parttype=partTypeExpr)? (isExtended=KW_EXTENDED)? -> ^(TOK_SHOWLOCKS $parttype? $isExtended?)
)
- | KW_SHOW (showOptions=KW_FORMATTED)? (KW_INDEX|KW_INDEXES) KW_ON showStmtIdentifier ((KW_FROM|KW_IN) db_name=identifier)?
- -> ^(TOK_SHOWINDEXES showStmtIdentifier $showOptions? $db_name?)
| KW_SHOW KW_COMPACTIONS -> ^(TOK_SHOW_COMPACTIONS)
| KW_SHOW KW_TRANSACTIONS -> ^(TOK_SHOW_TRANSACTIONS)
| KW_SHOW KW_CONF StringLiteral -> ^(TOK_SHOWCONF StringLiteral)
@@ -1839,7 +1735,6 @@ privilegeType
| KW_UPDATE -> ^(TOK_PRIV_ALTER_DATA)
| KW_CREATE -> ^(TOK_PRIV_CREATE)
| KW_DROP -> ^(TOK_PRIV_DROP)
- | KW_INDEX -> ^(TOK_PRIV_INDEX)
| KW_LOCK -> ^(TOK_PRIV_LOCK)
| KW_SELECT -> ^(TOK_PRIV_SELECT)
| KW_SHOW_DATABASE -> ^(TOK_PRIV_SHOW_DATABASE)
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java
deleted file mode 100644
index 22b6697..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.parse;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Index;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.optimizer.IndexUtils;
-import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
-import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.ql.session.LineageState;
-
-import java.io.Serializable;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class IndexUpdater {
- private List<LoadTableDesc> loadTableWork;
- private HiveConf conf;
- // Assumes one instance of this + single-threaded compilation for each query.
- private Hive hive;
- private List<Task<? extends Serializable>> tasks;
- private Set<ReadEntity> inputs;
- private LineageState lineageState;
-
- public IndexUpdater(List<LoadTableDesc> loadTableWork, Set<ReadEntity> inputs, Configuration conf,
- LineageState lineageState) {
- this.loadTableWork = loadTableWork;
- this.inputs = inputs;
- this.conf = new HiveConf(conf, IndexUpdater.class);
- this.lineageState = lineageState;
- this.tasks = new LinkedList<Task<? extends Serializable>>();
- }
-
- public IndexUpdater(LoadTableDesc loadTableWork, Set<ReadEntity> inputs,
- Configuration conf) {
- this.loadTableWork = new LinkedList<LoadTableDesc>();
- this.loadTableWork.add(loadTableWork);
- this.conf = new HiveConf(conf, IndexUpdater.class);
- this.tasks = new LinkedList<Task<? extends Serializable>>();
- this.inputs = inputs;
- }
-
- public List<Task<? extends Serializable>> generateUpdateTasks() throws
- HiveException {
- hive = Hive.get(this.conf);
- for (LoadTableDesc ltd : loadTableWork) {
- TableDesc td = ltd.getTable();
- Table srcTable = hive.getTable(td.getTableName());
- List<Index> tblIndexes = IndexUtils.getAllIndexes(srcTable, (short)-1);
- Map<String, String> partSpec = ltd.getPartitionSpec();
- if (partSpec == null || partSpec.size() == 0) {
- //unpartitioned table, update whole index
- doIndexUpdate(tblIndexes);
- } else {
- doIndexUpdate(tblIndexes, partSpec);
- }
- }
- return tasks;
- }
-
- private void doIndexUpdate(List<Index> tblIndexes) throws HiveException {
- for (Index idx : tblIndexes) {
- StringBuilder sb = new StringBuilder();
- sb.append("ALTER INDEX ");
- sb.append(idx.getIndexName());
- sb.append(" ON ");
- sb.append(idx.getDbName()).append('.');
- sb.append(idx.getOrigTableName());
- sb.append(" REBUILD");
- compileRebuild(sb.toString());
- }
- }
-
- private void doIndexUpdate(List<Index> tblIndexes, Map<String, String>
- partSpec) throws HiveException {
- for (Index index : tblIndexes) {
- if (containsPartition(index, partSpec)) {
- doIndexUpdate(index, partSpec);
- }
- }
- }
-
- private void doIndexUpdate(Index index, Map<String, String> partSpec) {
- StringBuilder ps = new StringBuilder();
- boolean first = true;
- ps.append("(");
- for (String key : partSpec.keySet()) {
- if (!first) {
- ps.append(", ");
- } else {
- first = false;
- }
- ps.append(key);
- ps.append("=");
- ps.append(partSpec.get(key));
- }
- ps.append(")");
- StringBuilder sb = new StringBuilder();
- sb.append("ALTER INDEX ");
- sb.append(index.getIndexName());
- sb.append(" ON ");
- sb.append(index.getDbName()).append('.');
- sb.append(index.getOrigTableName());
- sb.append(" PARTITION ");
- sb.append(ps.toString());
- sb.append(" REBUILD");
- compileRebuild(sb.toString());
- }
-
- private void compileRebuild(String query) {
- Driver driver = new Driver(this.conf, lineageState);
- driver.compile(query, false);
- tasks.addAll(driver.getPlan().getRootTasks());
- inputs.addAll(driver.getPlan().getInputs());
- }
-
-
- private boolean containsPartition(Index index,
- Map<String, String> partSpec) throws HiveException {
- String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName());
- Table indexTable = hive.getTable(qualified[0], qualified[1]);
- List<Partition> parts = hive.getPartitions(indexTable, partSpec);
- return (parts == null || parts.size() == 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 3619763..cc66936 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -360,25 +360,7 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
statTask = TaskFactory.get(columnStatsWork, conf);
}
- // HIVE-3334 has been filed for load file with index auto update
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) {
- IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, getInputs(), conf);
- try {
- List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater.generateUpdateTasks();
-
- for (Task<? extends Serializable> updateTask : indexUpdateTasks) {
- //LOAD DATA will either have a copy & move or just a move,
- // we always want the update to be dependent on the move
- childTask.addDependentTask(updateTask);
- if (statTask != null) {
- updateTask.addDependentTask(statTask);
- }
- }
- } catch (HiveException e) {
- console.printInfo("WARNING: could not auto-update stale indexes, indexes are not out of sync");
- }
- }
- else if (statTask != null) {
+ if (statTask != null) {
childTask.addDependentTask(statTask);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
index 2e1f50e..34963ff 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
@@ -74,7 +74,6 @@ public final class SemanticAnalyzerFactory {
commandType.put(HiveParser.TOK_SHOW_CREATEDATABASE, HiveOperation.SHOW_CREATEDATABASE);
commandType.put(HiveParser.TOK_SHOW_CREATETABLE, HiveOperation.SHOW_CREATETABLE);
commandType.put(HiveParser.TOK_SHOWFUNCTIONS, HiveOperation.SHOWFUNCTIONS);
- commandType.put(HiveParser.TOK_SHOWINDEXES, HiveOperation.SHOWINDEXES);
commandType.put(HiveParser.TOK_SHOWPARTITIONS, HiveOperation.SHOWPARTITIONS);
commandType.put(HiveParser.TOK_SHOWLOCKS, HiveOperation.SHOWLOCKS);
commandType.put(HiveParser.TOK_SHOWDBLOCKS, HiveOperation.SHOWLOCKS);
@@ -90,10 +89,6 @@ public final class SemanticAnalyzerFactory {
commandType.put(HiveParser.TOK_CREATE_MATERIALIZED_VIEW, HiveOperation.CREATE_MATERIALIZED_VIEW);
commandType.put(HiveParser.TOK_DROPVIEW, HiveOperation.DROPVIEW);
commandType.put(HiveParser.TOK_DROP_MATERIALIZED_VIEW, HiveOperation.DROP_MATERIALIZED_VIEW);
- commandType.put(HiveParser.TOK_CREATEINDEX, HiveOperation.CREATEINDEX);
- commandType.put(HiveParser.TOK_DROPINDEX, HiveOperation.DROPINDEX);
- commandType.put(HiveParser.TOK_ALTERINDEX_REBUILD, HiveOperation.ALTERINDEX_REBUILD);
- commandType.put(HiveParser.TOK_ALTERINDEX_PROPERTIES, HiveOperation.ALTERINDEX_PROPS);
commandType.put(HiveParser.TOK_ALTERVIEW_PROPERTIES, HiveOperation.ALTERVIEW_PROPERTIES);
commandType.put(HiveParser.TOK_ALTERVIEW_DROPPROPERTIES, HiveOperation.ALTERVIEW_PROPERTIES);
commandType.put(HiveParser.TOK_ALTERVIEW_ADDPARTS, HiveOperation.ALTERTABLE_ADDPARTS);
@@ -299,8 +294,6 @@ public final class SemanticAnalyzerFactory {
case HiveParser.TOK_DESCTABLE:
case HiveParser.TOK_DESCFUNCTION:
case HiveParser.TOK_MSCK:
- case HiveParser.TOK_ALTERINDEX_REBUILD:
- case HiveParser.TOK_ALTERINDEX_PROPERTIES:
case HiveParser.TOK_SHOWDATABASES:
case HiveParser.TOK_SHOWTABLES:
case HiveParser.TOK_SHOWCOLUMNS:
@@ -310,7 +303,6 @@ public final class SemanticAnalyzerFactory {
case HiveParser.TOK_SHOW_CREATETABLE:
case HiveParser.TOK_SHOWFUNCTIONS:
case HiveParser.TOK_SHOWPARTITIONS:
- case HiveParser.TOK_SHOWINDEXES:
case HiveParser.TOK_SHOWLOCKS:
case HiveParser.TOK_SHOWDBLOCKS:
case HiveParser.TOK_SHOW_COMPACTIONS:
@@ -319,8 +311,6 @@ public final class SemanticAnalyzerFactory {
case HiveParser.TOK_SHOWCONF:
case HiveParser.TOK_SHOWVIEWS:
case HiveParser.TOK_SHOWMATERIALIZEDVIEWS:
- case HiveParser.TOK_CREATEINDEX:
- case HiveParser.TOK_DROPINDEX:
case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT:
case HiveParser.TOK_LOCKTABLE:
case HiveParser.TOK_UNLOCKTABLE:
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 92d29e3..3122db8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -220,21 +220,6 @@ public abstract class TaskCompiler {
.get(new MoveWork(null, null, ltd, null, false),
conf);
mvTask.add(tsk);
- // Check to see if we are stale'ing any indexes and auto-update them if we want
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) {
- IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, inputs, conf,
- queryState.getLineageState());
- try {
- List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater
- .generateUpdateTasks();
- for (Task<? extends Serializable> updateTask : indexUpdateTasks) {
- tsk.addDependentTask(updateTask);
- }
- } catch (HiveException e) {
- console
- .printInfo("WARNING: could not auto-update stale indexes, which are not in sync");
- }
- }
}
boolean oneLoadFileForCtas = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java
deleted file mode 100644
index a335495..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterIndexDesc.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.plan;
-
-import java.io.Serializable;
-import java.util.Map;
-
-import org.apache.hadoop.hive.ql.plan.Explain.Level;
-
-/**
- * AlterIndexDesc.
- *
- */
-@Explain(displayName = "Alter Index", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterIndexDesc extends DDLDesc implements Serializable {
- private static final long serialVersionUID = 1L;
- private String indexName;
- private String baseTable;
- private Map<String, String> partSpec; // partition specification of partitions touched
- private Map<String, String> props;
-
- /**
- * alterIndexTypes.
- *
- */
- public static enum AlterIndexTypes {
- UPDATETIMESTAMP,
- ADDPROPS};
-
- AlterIndexTypes op;
-
- public AlterIndexDesc() {
- }
-
- public AlterIndexDesc(AlterIndexTypes type) {
- this.op = type;
- }
-
- /**
- * @return the name of the index
- */
- @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getIndexName() {
- return indexName;
- }
-
- /**
- * @param indexName
- * the indexName to set
- */
- public void setIndexName(String indexName) {
- this.indexName = indexName;
- }
-
- /**
- * @return the baseTable
- */
- @Explain(displayName = "new name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getBaseTableName() {
- return baseTable;
- }
-
- /**
- * @param baseTable
- * the baseTable to set
- */
- public void setBaseTableName(String baseTable) {
- this.baseTable = baseTable;
- }
-
- /**
- * @return the partition spec
- */
- public Map<String, String> getSpec() {
- return partSpec;
- }
-
- /**
- * @param partSpec
- * the partition spec to set
- */
- public void setSpec(Map<String, String> partSpec) {
- this.partSpec = partSpec;
- }
-
- /**
- * @return the op
- */
- public AlterIndexTypes getOp() {
- return op;
- }
-
- /**
- * @param op
- * the op to set
- */
- public void setOp(AlterIndexTypes op) {
- this.op = op;
- }
-
- /**
- * @return the props
- */
- @Explain(displayName = "properties")
- public Map<String, String> getProps() {
- return props;
- }
-
- /**
- * @param props
- * the props to set
- */
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0d3cb45/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java
deleted file mode 100644
index c003ee5..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateIndexDesc.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.plan;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
-/**
- * create index descriptor
- */
-public class CreateIndexDesc extends DDLDesc implements Serializable {
-
- private static final long serialVersionUID = 1L;
- String tableName;
- String indexName;
- List<String> indexedCols;
- String indexTableName;
- boolean deferredRebuild;
- String inputFormat;
- String outputFormat;
- String serde;
- String storageHandler;
- String indexTypeHandlerClass;
- String location;
- Map<String, String> idxProps;
- Map<String, String> tblProps;
- Map<String, String> serdeProps;
- String collItemDelim;
- String fieldDelim;
- String fieldEscape;
- String lineDelim;
- String mapKeyDelim;
-
- String indexComment;
-
- public CreateIndexDesc() {
- super();
- }
-
- public CreateIndexDesc(String tableName, String indexName,
- List<String> indexedCols, String indexTableName, boolean deferredRebuild,
- String inputFormat, String outputFormat, String storageHandler,
- String typeName, String location, Map<String, String> idxProps, Map<String, String> tblProps,
- String serde, Map<String, String> serdeProps, String collItemDelim,
- String fieldDelim, String fieldEscape, String lineDelim,
- String mapKeyDelim, String indexComment) {
- super();
- this.tableName = tableName;
- this.indexName = indexName;
- this.indexedCols = indexedCols;
- this.indexTableName = indexTableName;
- this.deferredRebuild = deferredRebuild;
- this.inputFormat = inputFormat;
- this.outputFormat = outputFormat;
- this.serde = serde;
- this.storageHandler = storageHandler;
- this.indexTypeHandlerClass = typeName;
- this.location = location;
- this.idxProps = idxProps;
- this.tblProps = tblProps;
- this.serde = serde;
- this.serdeProps = serdeProps;
- this.collItemDelim = collItemDelim;
- this.fieldDelim = fieldDelim;
- this.fieldEscape = fieldEscape;
- this.lineDelim = lineDelim;
- this.mapKeyDelim = mapKeyDelim;
- this.indexComment = indexComment;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public String getIndexName() {
- return indexName;
- }
-
- public void setIndexName(String indexName) {
- this.indexName = indexName;
- }
-
- public List<String> getIndexedCols() {
- return indexedCols;
- }
-
- public void setIndexedCols(List<String> indexedCols) {
- this.indexedCols = indexedCols;
- }
-
- public String getIndexTableName() {
- return indexTableName;
- }
-
- public void setIndexTableName(String indexTableName) {
- this.indexTableName = indexTableName;
- }
-
- public boolean isDeferredRebuild() {
- return deferredRebuild;
- }
-
- public boolean getDeferredRebuild() {
- return deferredRebuild;
- }
-
- public void setDeferredRebuild(boolean deferredRebuild) {
- this.deferredRebuild = deferredRebuild;
- }
-
- public String getInputFormat() {
- return inputFormat;
- }
-
- public void setInputFormat(String inputFormat) {
- this.inputFormat = inputFormat;
- }
-
- public String getOutputFormat() {
- return outputFormat;
- }
-
- public void setOutputFormat(String outputFormat) {
- this.outputFormat = outputFormat;
- }
-
- public String getSerde() {
- return serde;
- }
-
- public void setSerde(String serde) {
- this.serde = serde;
- }
-
- public String getStorageHandler() {
- return storageHandler;
- }
-
- public void setStorageHandler(String storageHandler) {
- this.storageHandler = storageHandler;
- }
-
- public String getLocation() {
- return location;
- }
-
- public void setLocation(String location) {
- this.location = location;
- }
-
- public Map<String, String> getIdxProps() {
- return idxProps;
- }
-
- public void setIdxProps(Map<String, String> idxProps) {
- this.idxProps = idxProps;
- }
-
- public Map<String, String> getTblProps() {
- return tblProps;
- }
-
- public void setTblProps(Map<String, String> tblProps) {
- this.tblProps = tblProps;
- }
-
- public Map<String, String> getSerdeProps() {
- return serdeProps;
- }
-
- public void setSerdeProps(Map<String, String> serdeProps) {
- this.serdeProps = serdeProps;
- }
-
- public String getCollItemDelim() {
- return collItemDelim;
- }
-
- public void setCollItemDelim(String collItemDelim) {
- this.collItemDelim = collItemDelim;
- }
-
- public String getFieldDelim() {
- return fieldDelim;
- }
-
- public void setFieldDelim(String fieldDelim) {
- this.fieldDelim = fieldDelim;
- }
-
- public String getFieldEscape() {
- return fieldEscape;
- }
-
- public void setFieldEscape(String fieldEscape) {
- this.fieldEscape = fieldEscape;
- }
-
- public String getLineDelim() {
- return lineDelim;
- }
-
- public void setLineDelim(String lineDelim) {
- this.lineDelim = lineDelim;
- }
-
- public String getMapKeyDelim() {
- return mapKeyDelim;
- }
-
- public void setMapKeyDelim(String mapKeyDelim) {
- this.mapKeyDelim = mapKeyDelim;
- }
-
- public String getIndexTypeHandlerClass() {
- return indexTypeHandlerClass;
- }
-
- public void setIndexTypeHandlerClass(String indexTypeHandlerClass) {
- this.indexTypeHandlerClass = indexTypeHandlerClass;
- }
-
- public String getIndexComment() {
- return indexComment;
- }
-
- public void setIndexComment(String indexComment) {
- this.indexComment = indexComment;
- }
-
-}