You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jv...@apache.org on 2011/06/21 03:41:18 UTC
svn commit: r1137839 [1/4] - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/index/
java/org/apache/hadoop/hive/ql/index/bitmap/
java/org/apache/hadoop/hive/ql/index/compact/
java/org/apache/hadoop/hive/ql/io/...
Author: jvs
Date: Tue Jun 21 01:41:17 2011
New Revision: 1137839
URL: http://svn.apache.org/viewvc?rev=1137839&view=rev
Log:
HIVE-2036. Update bitmap indexes for automatic usage (Syed Albiz via jvs)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapInnerQuery.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapOuterQuery.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapQuery.java
hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto.q
hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q
hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto.q.out
hive/trunk/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q
hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out
hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out
hive/trunk/ql/src/test/results/clientpositive/index_bitmap3.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Jun 21 01:41:17 2011
@@ -481,6 +481,7 @@ public class ExecDriver extends Task<Map
}
if (work.getIndexIntermediateFile() != null) {
conf.set("hive.index.compact.file", work.getIndexIntermediateFile());
+ conf.set("hive.index.blockfilter.file", work.getIndexIntermediateFile());
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java Tue Jun 21 01:41:17 2011
@@ -118,14 +118,19 @@ public interface HiveIndexHandler extend
throws HiveException;
/**
- * Generate the list of tasks required to run an index sub-query for the
- * given predicate, using the given index
- * @param index
+ * Generate the list of tasks required to run an index optimized sub-query for the
+ * given predicate, using the given indexes. If multiple indexes are
+ * provided, it is up to the handler whether to use none, one, some or all of
+ * them. The supplied predicate may reference any of the columns from any of
+ * the indexes. If the handler decides to use more than one index, it is
+ * responsible for generating tasks to combine their search results
+ * (e.g. performing a JOIN on the result).
+ * @param indexes
* @param predicate
* @param parseContext
* @param queryContext contains results, such as query tasks and input configuration
*/
- void generateIndexQuery(Index index, ExprNodeDesc predicate,
+ void generateIndexQuery(List<Index> indexes, ExprNodeDesc predicate,
ParseContext pctx, HiveIndexQueryContext queryContext);
/**
@@ -136,4 +141,4 @@ public interface HiveIndexHandler extend
* @return true if query is within the bounds
*/
boolean checkQuerySize(long inputSize, HiveConf conf);
-}
\ No newline at end of file
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java Tue Jun 21 01:41:17 2011
@@ -42,6 +42,8 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
/**
* IndexPredicateAnalyzer decomposes predicates, separating the parts
@@ -52,6 +54,7 @@ import org.apache.hadoop.hive.ql.udf.gen
*/
public class IndexPredicateAnalyzer
{
+ private static final Log LOG = LogFactory.getLog(IndexPredicateAnalyzer.class.getName());
private Set<String> udfNames;
private Set<String> allowedColumnNames;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java Tue Jun 21 01:41:17 2011
@@ -21,10 +21,14 @@ package org.apache.hadoop.hive.ql.index.
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
+import java.util.HashMap;
+import java.util.Map;
import java.util.List;
import java.util.Set;
-import java.util.Map.Entry;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -33,19 +37,30 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.index.HiveIndexQueryContext;
+import org.apache.hadoop.hive.ql.index.HiveIndexedInputFormat;;
+import org.apache.hadoop.hive.ql.index.IndexMetadataChangeTask;
+import org.apache.hadoop.hive.ql.index.IndexMetadataChangeWork;
+import org.apache.hadoop.hive.ql.index.IndexPredicateAnalyzer;
+import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
import org.apache.hadoop.hive.ql.index.TableBasedIndexHandler;
+import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
-import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapQuery;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.ql.index.IndexMetadataChangeTask;
-import org.apache.hadoop.hive.ql.index.IndexMetadataChangeWork;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
/**
* Index handler for the bitmap index. Bitmap index uses an EWAH-compressed
@@ -53,6 +68,142 @@ import org.apache.hadoop.hive.ql.index.I
*/
public class BitmapIndexHandler extends TableBasedIndexHandler {
+ private Configuration configuration;
+ private static final Log LOG = LogFactory.getLog(BitmapIndexHandler.class.getName());
+
+ @Override
+ public void generateIndexQuery(List<Index> indexes, ExprNodeDesc predicate,
+ ParseContext pctx, HiveIndexQueryContext queryContext) {
+
+ Map<Index, ExprNodeDesc> indexPredicates = decomposePredicate(
+ predicate,
+ indexes,
+ queryContext);
+
+ if (indexPredicates == null) {
+ LOG.info("No decomposed predicate found");
+ queryContext.setQueryTasks(null);
+ return; // abort if we couldn't pull out anything from the predicate
+ }
+
+ // Build reentrant QL for index query
+ StringBuilder qlCommand = new StringBuilder("INSERT OVERWRITE DIRECTORY ");
+
+ String tmpFile = pctx.getContext().getMRTmpFileURI();
+ qlCommand.append( "\"" + tmpFile + "\" "); // QL includes " around file name
+ qlCommand.append("SELECT bucketname AS `_bucketname` , COLLECT_SET(offset) AS `_offsets` FROM ");
+ qlCommand.append("(SELECT `_bucketname` AS bucketname , `_offset` AS offset FROM ");
+
+ List<BitmapInnerQuery> iqs = new ArrayList<BitmapInnerQuery>(indexes.size());
+ int i = 0;
+ for (Index index : indexes) {
+ ExprNodeDesc indexPredicate = indexPredicates.get(index);
+ if (indexPredicate != null) {
+ iqs.add(new BitmapInnerQuery(
+ index.getIndexTableName(),
+ indexPredicate,
+ "ind" + i++));
+ }
+ }
+
+ BitmapQuery head = iqs.get(0);
+ for ( i = 1; i < iqs.size(); i++) {
+ head = new BitmapOuterQuery("oind"+i, head, iqs.get(i));
+ }
+ qlCommand.append(head.toString());
+ qlCommand.append(" WHERE NOT EWAH_BITMAP_EMPTY(" + head.getAlias() + ".`_bitmaps`) ) tmp_index GROUP BY bucketname");
+
+ // generate tasks from index query string
+ LOG.info("Generating tasks for re-entrant QL query: " + qlCommand.toString());
+ Driver driver = new Driver(pctx.getConf());
+ driver.compile(qlCommand.toString(), false);
+
+ // setup TableScanOperator to change input format for original query
+ queryContext.setIndexInputFormat(HiveIndexedInputFormat.class.getName());
+ queryContext.setIndexIntermediateFile(tmpFile);
+
+ queryContext.addAdditionalSemanticInputs(driver.getPlan().getInputs());
+ queryContext.setQueryTasks(driver.getPlan().getRootTasks());
+ }
+
+ /**
+ * Split the predicate into the piece we can deal with (pushed), and the one we can't (residual)
+ * @param predicate
+ * @param index
+ * @return
+ */
+ private Map<Index, ExprNodeDesc> decomposePredicate(ExprNodeDesc predicate, List<Index> indexes,
+ HiveIndexQueryContext queryContext) {
+
+ Map<Index, ExprNodeDesc> indexPredicates = new HashMap<Index, ExprNodeDesc>();
+ // compute overall residual
+ IndexPredicateAnalyzer analyzer = getIndexPredicateAnalyzer(indexes, queryContext.getQueryPartitions());
+ List<IndexSearchCondition> searchConditions = new ArrayList<IndexSearchCondition>();
+ ExprNodeDesc residualPredicate = analyzer.analyzePredicate(predicate, searchConditions);
+ // pass residual predicate back out for further processing
+ queryContext.setResidualPredicate(residualPredicate);
+
+ if (searchConditions.size() == 0) {
+ return null;
+ }
+
+ for (Index index : indexes) {
+ ArrayList<Index> in = new ArrayList<Index>(1);
+ in.add(index);
+ analyzer = getIndexPredicateAnalyzer(in, queryContext.getQueryPartitions());
+ searchConditions = new ArrayList<IndexSearchCondition>();
+ // split predicate into pushed (what we can handle), and residual (what we can't handle)
+ // pushed predicate from translateSearchConditions is stored for the current index
+ // This ensures that we apply all possible predicates to each index
+ analyzer.analyzePredicate(predicate, searchConditions);
+ if (searchConditions.size() == 0) {
+ indexPredicates.put(index, null);
+ } else {
+ indexPredicates.put(index, analyzer.translateSearchConditions(searchConditions));
+ }
+ }
+
+ return indexPredicates;
+ }
+
+ /**
+ * Instantiate a new predicate analyzer suitable for determining
+ * whether we can use an index, based on rules for indexes in
+ * WHERE clauses that we support
+ *
+ * @return preconfigured predicate analyzer for WHERE queries
+ */
+ private IndexPredicateAnalyzer getIndexPredicateAnalyzer(List<Index> indexes, Set<Partition> queryPartitions) {
+ IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();
+
+ analyzer.addComparisonOp(GenericUDFOPEqual.class.getName());
+ analyzer.addComparisonOp(GenericUDFOPLessThan.class.getName());
+ analyzer.addComparisonOp(GenericUDFOPEqualOrLessThan.class.getName());
+ analyzer.addComparisonOp(GenericUDFOPGreaterThan.class.getName());
+ analyzer.addComparisonOp(GenericUDFOPEqualOrGreaterThan.class.getName());
+
+ // only return results for columns in the list of indexes
+ for (Index index : indexes) {
+ List<FieldSchema> columnSchemas = index.getSd().getCols();
+ for (FieldSchema column : columnSchemas) {
+ analyzer.allowColumnName(column.getName());
+ }
+ }
+
+ // partitioned columns are treated as if they have indexes so that the partitions
+ // are used during the index query generation
+ for (Partition part : queryPartitions) {
+ if (part.getSpec().isEmpty()) {
+ continue; // empty partitions are from whole tables, so we don't want to add them in
+ }
+ for (String column : part.getSpec().keySet()) {
+ analyzer.allowColumnName(column);
+ }
+ }
+
+ return analyzer;
+ }
+
@Override
public void analyzeIndexDefinition(Table baseTable, Index index,
Table indexTable) throws HiveException {
@@ -78,6 +229,8 @@ public class BitmapIndexHandler extends
HiveConf conf = new HiveConf(getConf(), BitmapIndexHandler.class);
HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEROWOFFSET, true);
+ // Don't try to index optimize the query to build the index
+ HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEOPTINDEXFILTER, false);
String indexCols = HiveUtils.getUnparsedColumnNamesFromFieldSchema(indexField);
@@ -130,9 +283,9 @@ public class BitmapIndexHandler extends
command.append(",");
command.append(HiveUtils.unparseIdentifier(fieldSchema.getName()));
}
-
+
// Require clusterby ROWOFFSET if map-size aggregation is off.
- if (!configuration.get("hive.map.aggr", null).equals("true")) {
+ if (!conf.get("hive.map.aggr", null).equals("true")) {
command.append(" CLUSTER BY ");
command.append(VirtualColumn.ROWOFFSET.getName());
}
@@ -151,4 +304,18 @@ public class BitmapIndexHandler extends
return rootTask;
}
+
+ @Override
+ /**
+ * No lower bound on bitmap index query size, so this will always return true
+ */
+ public boolean checkQuerySize(long querySize, HiveConf hiveConf) {
+ return true;
+ }
+
+ @Override
+ public boolean usesIndexTable() {
+ return true;
+ }
+
}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapInnerQuery.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapInnerQuery.java?rev=1137839&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapInnerQuery.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapInnerQuery.java Tue Jun 21 01:41:17 2011
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index.bitmap;
+
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapQuery;
+
+/**
+ * Representation of inner bitmap index SELECT query that scans bitmap index
+ * tables for a pushed predicate
+ */
+public class BitmapInnerQuery implements BitmapQuery {
+ private String tableName;
+ private ExprNodeDesc predicate;
+ private String alias;
+ private String queryStr;
+
+ public BitmapInnerQuery(String tableName, ExprNodeDesc predicate, String alias) {
+ this.tableName = tableName;
+ this.predicate = predicate;
+ this.alias = alias;
+ constructQueryStr();
+ }
+
+ /**
+ * Return a string representation of the query string for compilation
+ */
+ public String toString() {
+ return queryStr;
+ }
+
+ /**
+ * Construct a string representation of the query to be compiled
+ */
+ private void constructQueryStr() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("(SELECT * FROM ");
+ sb.append(HiveUtils.unparseIdentifier(tableName));
+ sb.append(" WHERE ");
+ sb.append(predicate.getExprString());
+ sb.append(") ");
+ sb.append(alias);
+ queryStr = sb.toString();
+ }
+
+ /**
+ * Return the assigned alias of the SELECT statement
+ */
+ public String getAlias() {
+ return alias;
+ }
+
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapOuterQuery.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapOuterQuery.java?rev=1137839&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapOuterQuery.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapOuterQuery.java Tue Jun 21 01:41:17 2011
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index.bitmap;
+
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapQuery;
+
+/**
+ * Representation of the outer query on bitmap indexes that JOINs the result of
+ * inner SELECT scans on bitmap indexes (represented in BitmapQuery objects)
+ * using EWAH_* bitwise operations
+ */
+public class BitmapOuterQuery implements BitmapQuery {
+ private String alias;
+ private BitmapQuery lhs;
+ private BitmapQuery rhs;
+ private String queryStr;
+
+ public BitmapOuterQuery(String alias, BitmapQuery lhs, BitmapQuery rhs) {
+ this.alias = alias;
+ this.lhs = lhs;
+ this.rhs = rhs;
+ constructQueryStr();
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ /**
+ * Return a string representation of the query for compilation
+ */
+ public String toString() {
+ return queryStr;
+ }
+
+ /**
+ * Construct a string representation of the query to be compiled
+ */
+ private void constructQueryStr() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("(SELECT ");
+ sb.append(lhs.getAlias());
+ sb.append(".`_bucketname`, ");
+ sb.append(rhs.getAlias());
+ sb.append(".`_offset`, ");
+ sb.append("EWAH_BITMAP_AND(");
+ sb.append(lhs.getAlias());
+ sb.append(".`_bitmaps`, ");
+ sb.append(rhs.getAlias());
+ sb.append(".`_bitmaps`) AS `_bitmaps` FROM ");
+ sb.append(lhs.toString());
+ sb.append(" JOIN ");
+ sb.append(rhs.toString());
+ sb.append(" ON ");
+ sb.append(lhs.getAlias());
+ sb.append(".`_bucketname` = ");
+ sb.append(rhs.getAlias());
+ sb.append(".`_bucketname` AND ");
+ sb.append(lhs.getAlias());
+ sb.append(".`_offset` = ");
+ sb.append(rhs.getAlias());
+ sb.append(".`_offset`) ");
+ sb.append(this.alias);
+ queryStr = sb.toString();
+ }
+
+}
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapQuery.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapQuery.java?rev=1137839&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapQuery.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapQuery.java Tue Jun 21 01:41:17 2011
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.index.bitmap;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+
+/**
+ * Generic interface to representations of queries on bitmap indexes
+ */
+public interface BitmapQuery {
+ public String getAlias();
+
+ public String toString();
+}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java Tue Jun 21 01:41:17 2011
@@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler.DecomposedPredicate;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
@@ -127,7 +128,10 @@ public class CompactIndexHandler extends
command.append(" GROUP BY ");
command.append(indexCols + ", " + VirtualColumn.FILENAME.getName());
- Driver driver = new Driver(new HiveConf(getConf(), CompactIndexHandler.class));
+ HiveConf builderConf = new HiveConf(getConf(), CompactIndexHandler.class);
+ // Don't try to index optimize the query to build the index
+ HiveConf.setBoolVar(builderConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER, false);
+ Driver driver = new Driver(builderConf);
driver.compile(command.toString());
Task<?> rootTask = driver.getPlan().getRootTasks().get(0);
@@ -143,9 +147,10 @@ public class CompactIndexHandler extends
}
@Override
- public void generateIndexQuery(Index index, ExprNodeDesc predicate,
+ public void generateIndexQuery(List<Index> indexes, ExprNodeDesc predicate,
ParseContext pctx, HiveIndexQueryContext queryContext) {
+ Index index = indexes.get(0);
DecomposedPredicate decomposedPredicate = decomposePredicate(predicate, index,
queryContext.getQueryPartitions());
@@ -235,9 +240,8 @@ public class CompactIndexHandler extends
if (part.getSpec().isEmpty()) {
continue; // empty partitions are from whole tables, so we don't want to add them in
}
- List<FieldSchema> partitionColumns = part.getCols();
- for (FieldSchema column : partitionColumns) {
- analyzer.allowColumnName(column.getName());
+ for (String column : part.getSpec().keySet()) {
+ analyzer.allowColumnName(column);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java Tue Jun 21 01:41:17 2011
@@ -262,6 +262,7 @@ public final class HiveFileFormatUtils {
boolean ignoreSchema) throws IOException {
PartitionDesc part = doGetPartitionDescFromPath(pathToPartitionInfo, dir);
+
if (part == null
&& (ignoreSchema || (dir.toUri().getScheme() == null || dir.toUri().getScheme().trim()
.equals("")))) {
@@ -281,7 +282,6 @@ public final class HiveFileFormatUtils {
}
part = doGetPartitionDescFromPath(newPathToPartitionInfo, dir);
}
-
if (part != null) {
return part;
} else {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java Tue Jun 21 01:41:17 2011
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.parse.P
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
+import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.plan.MapredWork;
/**
@@ -74,17 +75,24 @@ public class IndexWhereProcessor impleme
@Override
/**
- * Process a node of the operator tree. This matches on the rule in IndexWhereTaskDispatcher
+ * Process a node of the operator tree. This matches on the rule in IndexWhereTaskDispatcher
*/
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- FilterOperator operator = (FilterOperator) nd;
- FilterDesc operatorDesc = operator.getConf();
- ExprNodeDesc predicate = operatorDesc.getPredicate();
+ TableScanOperator operator = (TableScanOperator) nd;
+ List<Node> opChildren = operator.getChildren();
+ TableScanDesc operatorDesc = operator.getConf();
+ ExprNodeDesc predicate = operatorDesc.getFilterExpr();
IndexWhereProcCtx context = (IndexWhereProcCtx) procCtx;
ParseContext pctx = context.getParseContext();
+ LOG.info("Processing predicate for index optimization");
+ if (predicate == null) {
+ LOG.info("null predicate pushed down");
+ return null;
+ }
+ LOG.info(predicate.getExprString());
// check if we have indexes on all partitions in this table scan
Set<Partition> queryPartitions;
@@ -108,18 +116,46 @@ public class IndexWhereProcessor impleme
Map<Index, HiveIndexQueryContext> queryContexts = new HashMap<Index, HiveIndexQueryContext>();
Collection<List<Index>> tableIndexes = indexes.values();
for (List<Index> indexesOnTable : tableIndexes) {
+ List<List<Index>> indexesByType = new ArrayList<List<Index>>();
for (Index index : indexesOnTable) {
- HiveIndexQueryContext queryContext = new HiveIndexQueryContext();
- queryContext.setQueryPartitions(queryPartitions);
- rewriteForIndex(predicate, index, pctx, currentTask, queryContext);
- List<Task<?>> indexTasks = queryContext.getQueryTasks();
+ boolean added = false;
+ for (List<Index> indexType : indexesByType) {
+ if (indexType.isEmpty()) {
+ indexType.add(index);
+ added = true;
+ } else if (indexType.get(0).getIndexHandlerClass().equals(
+ index.getIndexHandlerClass())) {
+ indexType.add(index);
+ added = true;
+ break;
+ }
+ }
+ if (!added) {
+ List<Index> newType = new ArrayList<Index>();
+ newType.add(index);
+ indexesByType.add(newType);
+ }
+ }
- if (indexTasks != null && indexTasks.size() > 0) {
- queryContexts.put(index, queryContext);
+ // choose index type with most indexes of the same type on the table
+ // TODO HIVE-2130 This would be a good place for some sort of cost based choice?
+ List<Index> bestIndexes = indexesByType.get(0);
+ for (List<Index> indexTypes : indexesByType) {
+ if (bestIndexes.size() < indexTypes.size()) {
+ bestIndexes = indexTypes;
}
}
- }
+ // rewrite index queries for the chosen index type
+ HiveIndexQueryContext queryContext = new HiveIndexQueryContext();
+ queryContext.setQueryPartitions(queryPartitions);
+ rewriteForIndexes(predicate, bestIndexes, pctx, currentTask, queryContext);
+ List<Task<?>> indexTasks = queryContext.getQueryTasks();
+
+ if (indexTasks != null && indexTasks.size() > 0) {
+ queryContexts.put(bestIndexes.get(0), queryContext);
+ }
+ }
// choose an index rewrite to use
if (queryContexts.size() > 0) {
// TODO HIVE-2130 This would be a good place for some sort of cost based choice?
@@ -137,7 +173,6 @@ public class IndexWhereProcessor impleme
// modify inputs based on index query
Set<ReadEntity> inputs = pctx.getSemanticInputs();
inputs.addAll(queryContext.getAdditionalSemanticInputs());
-
List<Task<?>> chosenRewrite = queryContext.getQueryTasks();
// add dependencies so index query runs first
@@ -157,11 +192,14 @@ public class IndexWhereProcessor impleme
* @param task original task before rewrite
* @param queryContext stores return values
*/
- private void rewriteForIndex(ExprNodeDesc predicate, Index index,
+ private void rewriteForIndexes(ExprNodeDesc predicate, List<Index> indexes,
ParseContext pctx, Task<MapredWork> task,
HiveIndexQueryContext queryContext)
throws SemanticException {
HiveIndexHandler indexHandler;
+ // All indexes in the list are of the same type, and therefore can use the
+ // same handler to generate the index query tasks
+ Index index = indexes.get(0);
try {
indexHandler = HiveUtils.getIndexHandler(pctx.getConf(), index.getIndexHandlerClass());
} catch (HiveException e) {
@@ -182,7 +220,7 @@ public class IndexWhereProcessor impleme
}
// use the IndexHandler to generate the index query
- indexHandler.generateIndexQuery(index, predicate, pctx, queryContext);
+ indexHandler.generateIndexQuery(indexes, predicate, pctx, queryContext);
// TODO HIVE-2115 use queryContext.residualPredicate to process residual predicate
return;
@@ -197,9 +235,8 @@ public class IndexWhereProcessor impleme
* @param operator
* @return partitions used by query. null if they do not exist in index table
*/
- private Set<Partition> checkPartitionsCoveredByIndex(FilterOperator operator, ParseContext pctx)
+ private Set<Partition> checkPartitionsCoveredByIndex(TableScanOperator tableScan, ParseContext pctx)
throws HiveException {
- TableScanOperator tableScan = (TableScanOperator) operator.getParentOperators().get(0);
Hive hive = Hive.get(pctx.getConf());
// make sure each partition exists on the index table
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java Tue Jun 21 01:41:17 2011
@@ -30,6 +30,7 @@ import java.util.Stack;
import org.apache.hadoop.hive.metastore.api.Index;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler;
+import org.apache.hadoop.hive.ql.index.bitmap.BitmapIndexHandler;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
@@ -105,6 +106,7 @@ public class IndexWhereTaskDispatcher im
List<String> supportedIndexes = new ArrayList<String>();
supportedIndexes.add(CompactIndexHandler.class.getName());
+ supportedIndexes.add(BitmapIndexHandler.class.getName());
// query the metastore to know what columns we have indexed
Collection<Table> topTables = pctx.getTopToTable().values();
@@ -122,8 +124,9 @@ public class IndexWhereTaskDispatcher im
return null;
}
- // FIL% is a filter operator, a WHERE shows up as a filter on a table scan operator (TS%)
- operatorRules.put(new RuleRegExp("RULEWhere", "TS%FIL%"), new IndexWhereProcessor(indexes));
+ // We set the pushed predicate from the WHERE clause as the filter expr on
+ // all table scan operators, so we look for table scan operators(TS%)
+ operatorRules.put(new RuleRegExp("RULEWhere", "TS%"), new IndexWhereProcessor(indexes));
return operatorRules;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java Tue Jun 21 01:41:17 2011
@@ -24,6 +24,8 @@ import java.util.List;
import java.util.Map;
import java.util.Stack;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
@@ -48,6 +50,9 @@ import org.apache.hadoop.hive.ql.plan.Ex
* pushdown optimization for the given operator
*/
public final class ExprWalkerProcFactory {
+
+ private static final Log LOG = LogFactory
+ .getLog(ExprWalkerProcFactory.class.getName());
/**
* ColumnExprProcessor.
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Tue Jun 21 01:41:17 2011
@@ -450,6 +450,12 @@ public final class OpProcFactory {
TableScanDesc tableScanDesc = tableScanOp.getConf();
Table tbl = owi.getParseContext().getTopToTable().get(tableScanOp);
+ if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEOPTINDEXFILTER)) {
+ // attach the original predicate to the table scan operator for index
+ // optimizations that require the pushed predicate before pcr & later
+ // optimizations are applied
+ tableScanDesc.setFilterExpr(originalPredicate);
+ }
if (!tbl.isNonNative()) {
return originalPredicate;
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q Tue Jun 21 01:41:17 2011
@@ -6,7 +6,7 @@ SET hive.input.format=org.apache.hadoop.
SET hive.optimize.index.filter=true;
SET hive.optimize.index.filter.compact.minsize=0;
-EXPLAIN SELECT key, value FROM srcpart WHERE key=86 ORDER BY key;
-SELECT key, value FROM srcpart WHERE key=86 ORDER BY key;
+EXPLAIN SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key;
+SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key;
DROP INDEX src_part_index ON srcpart;
Added: hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto.q?rev=1137839&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto.q Tue Jun 21 01:41:17 2011
@@ -0,0 +1,52 @@
+-- try the query without indexing, with manual indexing, and with automatic indexing
+-- without indexing
+SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key;
+
+-- create indices
+EXPLAIN
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD;
+EXPLAIN
+CREATE INDEX src2_index ON TABLE src(value) as 'BITMAP' WITH DEFERRED REBUILD;
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+CREATE INDEX src1_index ON TABLE src(key) as 'BITMAP' WITH DEFERRED REBUILD;
+CREATE INDEX src2_index ON TABLE src(value) as 'BITMAP' WITH DEFERRED REBUILD;
+ALTER INDEX src1_index ON src REBUILD;
+ALTER INDEX src2_index ON src REBUILD;
+SELECT * FROM default__src_src1_index__ ORDER BY key;
+SELECT * FROM default__src_src2_index__ ORDER BY value;
+
+
+-- manual indexing
+EXPLAIN
+SELECT a.bucketname AS `_bucketname`, COLLECT_SET(a.offset) as `_offsets`
+FROM (SELECT `_bucketname` AS bucketname, `_offset` AS offset, `_bitmaps` AS bitmaps FROM default__src_src1_index__
+ WHERE key = 0) a
+ JOIN
+ (SELECT `_bucketname` AS bucketname, `_offset` AS offset, `_bitmaps` AS bitmaps FROM default__src_src2_index__
+ WHERE value = "val_0") b
+ ON
+ a.bucketname = b.bucketname AND a.offset = b.offset WHERE NOT
+EWAH_BITMAP_EMPTY(EWAH_BITMAP_AND(a.bitmaps, b.bitmaps)) GROUP BY a.bucketname;
+
+INSERT OVERWRITE DIRECTORY "/tmp/index_result"
+SELECT a.bucketname AS `_bucketname`, COLLECT_SET(a.offset) as `_offsets`
+FROM (SELECT `_bucketname` AS bucketname, `_offset` AS offset, `_bitmaps` AS bitmaps FROM default__src_src1_index__
+ WHERE key = 0) a
+ JOIN
+ (SELECT `_bucketname` AS bucketname, `_offset` AS offset, `_bitmaps` AS bitmaps FROM default__src_src2_index__
+ WHERE value = "val_0") b
+ ON
+ a.bucketname = b.bucketname AND a.offset = b.offset WHERE NOT
+EWAH_BITMAP_EMPTY(EWAH_BITMAP_AND(a.bitmaps, b.bitmaps)) GROUP BY a.bucketname;
+
+SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key;
+
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+SET hive.optimize.index.filter=true;
+SELECT key, value FROM src WHERE key=0 AND value = "val_0" ORDER BY key;
+
+DROP INDEX src1_index ON src;
+DROP INDEX src2_index ON src;
+
Added: hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q?rev=1137839&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q Tue Jun 21 01:41:17 2011
@@ -0,0 +1,11 @@
+-- test automatic use of index on table with partitions
+CREATE INDEX src_part_index ON TABLE srcpart(key) as 'BITMAP' WITH DEFERRED REBUILD;
+ALTER INDEX src_part_index ON srcpart REBUILD;
+
+SET hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+SET hive.optimize.index.filter=true;
+
+EXPLAIN SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key;
+SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key;
+
+DROP INDEX src_part_index ON srcpart;
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto.q.out Tue Jun 21 01:41:17 2011
@@ -4,14 +4,14 @@ PREHOOK: query: -- try the query without
SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-48-49_890_8915143803732182055/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-16_11-31-55_780_4816519858356075154/-mr-10000
POSTHOOK: query: -- try the query without indexing, with manual indexing, and with automatic indexing
-- without indexing
SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-48-49_890_8915143803732182055/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-16_11-31-55_780_4816519858356075154/-mr-10000
82 val_82
83 val_83
83 val_83
@@ -123,11 +123,11 @@ STAGE PLANS:
PREHOOK: query: SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-49-24_225_5627614687777405478/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-16_11-32-19_097_338721874120393596/-mr-10000
POSTHOOK: query: SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-49-24_225_5627614687777405478/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-16_11-32-19_097_338721874120393596/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -178,6 +178,9 @@ STAGE PLANS:
default__src_src_index__
TableScan
alias: default__src_src_index__
+ filterExpr:
+ expr: ((key > 80) and (key < 100))
+ type: boolean
Filter Operator
predicate:
expr: ((key > 80) and (key < 100))
@@ -207,13 +210,13 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-49-31_422_3293105246070967294/-ext-10000
+ destination: file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-16_11-32-26_654_6395896159951974498/-ext-10000
Stage: Stage-2
Move Operator
files:
hdfs directory: true
- destination: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-49-31_020_3781399652075665616/-mr-10002
+ destination: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-16_11-32-26_508_4648561266114809466/-mr-10002
Stage: Stage-1
Map Reduce
@@ -221,6 +224,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: ((key > 80) and (key < 100))
+ type: boolean
Filter Operator
predicate:
expr: ((key > 80) and (key < 100))
@@ -259,7 +265,7 @@ STAGE PLANS:
Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-49-31_422_3293105246070967294/-ext-10001
+ file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-16_11-32-26_654_6395896159951974498/-ext-10001
File Output Operator
compressed: false
GlobalTableId: 0
@@ -276,12 +282,12 @@ PREHOOK: query: SELECT key, value FROM s
PREHOOK: type: QUERY
PREHOOK: Input: default@default__src_src_index__
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-49-31_665_4650797773210786014/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-16_11-32-26_783_5543392894671342055/-mr-10000
POSTHOOK: query: SELECT key, value FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__src_src_index__
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-49-31_665_4650797773210786014/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-16_11-32-26_783_5543392894671342055/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_file_format.q.out Tue Jun 21 01:41:17 2011
@@ -41,6 +41,9 @@ STAGE PLANS:
default__src_src_index__
TableScan
alias: default__src_src_index__
+ filterExpr:
+ expr: (key = 86)
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
@@ -70,13 +73,13 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-34-34_553_2035287069819677240/-ext-10000
+ destination: file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-16-40_411_4815505312955631583/-ext-10000
Stage: Stage-2
Move Operator
files:
hdfs directory: true
- destination: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-34-33_550_5081238212103104306/-mr-10002
+ destination: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-16-40_275_2821007696524201513/-mr-10002
Stage: Stage-1
Map Reduce
@@ -84,6 +87,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: (key = 86)
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
@@ -122,7 +128,7 @@ STAGE PLANS:
Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-34-34_553_2035287069819677240/-ext-10001
+ file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-16-40_411_4815505312955631583/-ext-10001
File Output Operator
compressed: false
GlobalTableId: 0
@@ -139,12 +145,12 @@ PREHOOK: query: SELECT key, value FROM s
PREHOOK: type: QUERY
PREHOOK: Input: default@default__src_src_index__
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-34-35_027_2206378732102210462/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-16-40_552_3841543983008204664/-mr-10000
POSTHOOK: query: SELECT key, value FROM src WHERE key=86 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__src_src_index__
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-34-35_027_2206378732102210462/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-16-40_552_3841543983008204664/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -175,6 +181,9 @@ STAGE PLANS:
default__src_src_index__
TableScan
alias: default__src_src_index__
+ filterExpr:
+ expr: (key = 86)
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
@@ -204,13 +213,13 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-34-55_926_5572346744193019133/-ext-10000
+ destination: file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-16-55_117_7145184773994968051/-ext-10000
Stage: Stage-2
Move Operator
files:
hdfs directory: true
- destination: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-34-55_774_5724664415454979907/-mr-10002
+ destination: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-16-54_980_8246330207973926102/-mr-10002
Stage: Stage-1
Map Reduce
@@ -218,6 +227,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: (key = 86)
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
@@ -256,7 +268,7 @@ STAGE PLANS:
Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-34-55_926_5572346744193019133/-ext-10001
+ file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-16-55_117_7145184773994968051/-ext-10001
File Output Operator
compressed: false
GlobalTableId: 0
@@ -273,12 +285,12 @@ PREHOOK: query: SELECT key, value FROM s
PREHOOK: type: QUERY
PREHOOK: Input: default@default__src_src_index__
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-34-56_142_8911654156491576181/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-16-55_246_5645043276355070579/-mr-10000
POSTHOOK: query: SELECT key, value FROM src WHERE key=86 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__src_src_index__
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-34-56_142_8911654156491576181/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-16-55_246_5645043276355070579/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_multiple.q.out Tue Jun 21 01:41:17 2011
@@ -64,6 +64,9 @@ STAGE PLANS:
default__src_src_key_index__
TableScan
alias: default__src_src_key_index__
+ filterExpr:
+ expr: (key = 86)
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
@@ -93,13 +96,13 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-05-01_12-04-18_443_8927626385044275013/-ext-10000
+ destination: file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-17-27_086_2000600421786753633/-ext-10000
Stage: Stage-2
Move Operator
files:
hdfs directory: true
- destination: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-05-01_12-04-16_825_4915849285508089033/-mr-10002
+ destination: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-17-26_945_142379732782210508/-mr-10002
Stage: Stage-1
Map Reduce
@@ -107,6 +110,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: (key = 86)
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
@@ -145,7 +151,7 @@ STAGE PLANS:
Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-05-01_12-04-18_443_8927626385044275013/-ext-10001
+ file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-17-27_086_2000600421786753633/-ext-10001
File Output Operator
compressed: false
GlobalTableId: 0
@@ -162,12 +168,12 @@ PREHOOK: query: SELECT key, value FROM s
PREHOOK: type: QUERY
PREHOOK: Input: default@default__src_src_key_index__
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-05-01_12-04-19_040_4079835930725904895/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-17-27_239_7970970441956583684/-mr-10000
POSTHOOK: query: SELECT key, value FROM src WHERE key=86 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@default__src_src_key_index__
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-05-01_12-04-19_040_4079835930725904895/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-17-27_239_7970970441956583684/-mr-10000
POSTHOOK: Lineage: default__src_src_key_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_key_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_key_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_partitioned.q.out Tue Jun 21 01:41:17 2011
@@ -27,15 +27,15 @@ POSTHOOK: Output: default@default__srcpa
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12)._bucketname SIMPLE [(srcpart)srcpart.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12)._offsets EXPRESSION [(srcpart)srcpart.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
-PREHOOK: query: EXPLAIN SELECT key, value FROM srcpart WHERE key=86 ORDER BY key
+PREHOOK: query: EXPLAIN SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key
PREHOOK: type: QUERY
-POSTHOOK: query: EXPLAIN SELECT key, value FROM srcpart WHERE key=86 ORDER BY key
+POSTHOOK: query: EXPLAIN SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12)._bucketname SIMPLE [(srcpart)srcpart.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12)._offsets EXPRESSION [(srcpart)srcpart.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
ABSTRACT SYNTAX TREE:
- (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (= (TOK_TABLE_OR_COL key) 86)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)))))
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL key) 86) (= (TOK_TABLE_OR_COL ds) '2008-04-09'))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)))))
STAGE DEPENDENCIES:
Stage-3 is a root stage
@@ -53,13 +53,16 @@ STAGE PLANS:
default__srcpart_src_part_index__
TableScan
alias: default__srcpart_src_part_index__
+ filterExpr:
+ expr: ((key = 86) and (ds = '2008-04-09'))
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
type: boolean
Filter Operator
predicate:
- expr: (key = 86)
+ expr: ((key = 86) and (ds = '2008-04-09'))
type: boolean
Select Operator
expressions:
@@ -82,13 +85,13 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- destination: file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-42-19_218_3331226633297238280/-ext-10000
+ destination: file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-00-25_190_7313794997314528333/-ext-10000
Stage: Stage-2
Move Operator
files:
hdfs directory: true
- destination: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-42-17_934_3278921987899048358/-mr-10002
+ destination: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-00-24_901_7521678807552233457/-mr-10002
Stage: Stage-1
Map Reduce
@@ -96,13 +99,16 @@ STAGE PLANS:
srcpart
TableScan
alias: srcpart
+ filterExpr:
+ expr: ((key = 86) and (ds = '2008-04-09'))
+ type: boolean
Filter Operator
predicate:
expr: (key = 86)
type: boolean
Filter Operator
predicate:
- expr: (key = 86)
+ expr: ((key = 86) and (ds = '2008-04-09'))
type: boolean
Select Operator
expressions:
@@ -134,7 +140,7 @@ STAGE PLANS:
Stage: Stage-4
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/rmelick/hive/build/ql/scratchdir/hive_2011-04-15_22-42-19_218_3331226633297238280/-ext-10001
+ file:/Users/salbiz/dev/hive/build/ql/scratchdir/hive_2011-06-15_16-00-25_190_7313794997314528333/-ext-10001
File Output Operator
compressed: false
GlobalTableId: 0
@@ -147,35 +153,25 @@ STAGE PLANS:
limit: -1
-PREHOOK: query: SELECT key, value FROM srcpart WHERE key=86 ORDER BY key
+PREHOOK: query: SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key
PREHOOK: type: QUERY
-PREHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-08/hr=11
-PREHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-08/hr=12
PREHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-09/hr=11
PREHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-09/hr=12
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-42-19_839_770753444589987409/-mr-10000
-POSTHOOK: query: SELECT key, value FROM srcpart WHERE key=86 ORDER BY key
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-00-25_389_6851974893264734752/-mr-10000
+POSTHOOK: query: SELECT key, value FROM srcpart WHERE key=86 AND ds='2008-04-09' ORDER BY key
POSTHOOK: type: QUERY
-POSTHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-08/hr=11
-POSTHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-08/hr=12
POSTHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-09/hr=11
POSTHOOK: Input: default@default__srcpart_src_part_index__@ds=2008-04-09/hr=12
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
-POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-15_22-42-19_839_770753444589987409/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-00-25_389_6851974893264734752/-mr-10000
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12)._bucketname SIMPLE [(srcpart)srcpart.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12)._offsets EXPRESSION [(srcpart)srcpart.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__srcpart_src_part_index__ PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
86 val_86
86 val_86
-86 val_86
-86 val_86
PREHOOK: query: DROP INDEX src_part_index ON srcpart
PREHOOK: type: DROPINDEX
POSTHOOK: query: DROP INDEX src_part_index ON srcpart
Modified: hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out?rev=1137839&r1=1137838&r2=1137839&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/index_auto_unused.q.out Tue Jun 21 01:41:17 2011
@@ -40,6 +40,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: ((key > 80) and (key < 100))
+ type: boolean
Filter Operator
predicate:
expr: ((key > 80) and (key < 100))
@@ -83,11 +86,11 @@ STAGE PLANS:
PREHOOK: query: SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-13-34_118_2447968768171073218/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-25_374_4374810764646255651/-mr-10000
POSTHOOK: query: SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-13-34_118_2447968768171073218/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-25_374_4374810764646255651/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -133,6 +136,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: ((key > 80) and (key < 100))
+ type: boolean
Filter Operator
predicate:
expr: ((key > 80) and (key < 100))
@@ -176,11 +182,11 @@ STAGE PLANS:
PREHOOK: query: SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-13-42_868_8608290458938388381/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-33_059_3495425540612459913/-mr-10000
POSTHOOK: query: SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-13-42_868_8608290458938388381/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-33_059_3495425540612459913/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -226,6 +232,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: ((key < 10) or (key > 480))
+ type: boolean
Filter Operator
predicate:
expr: ((key < 10) or (key > 480))
@@ -269,11 +278,11 @@ STAGE PLANS:
PREHOOK: query: SELECT * FROM src WHERE key < 10 OR key > 480 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-13-51_158_1683546756150736409/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-41_214_8813442842757881648/-mr-10000
POSTHOOK: query: SELECT * FROM src WHERE key < 10 OR key > 480 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-13-51_158_1683546756150736409/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-41_214_8813442842757881648/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -363,6 +372,9 @@ STAGE PLANS:
src
TableScan
alias: src
+ filterExpr:
+ expr: ((key > 80) and (key < 100))
+ type: boolean
Filter Operator
predicate:
expr: ((key > 80) and (key < 100))
@@ -406,11 +418,11 @@ STAGE PLANS:
PREHOOK: query: SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@src
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-14-14_046_9000844070247643784/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-56_659_721280297272334789/-mr-10000
POSTHOOK: query: SELECT * FROM src WHERE key > 80 AND key < 100 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-14-14_046_9000844070247643784/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-09-56_659_721280297272334789/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
@@ -502,6 +514,9 @@ STAGE PLANS:
srcpart
TableScan
alias: srcpart
+ filterExpr:
+ expr: (((ds = '2008-04-09') and (hr = 12)) and (key < 10))
+ type: boolean
Filter Operator
predicate:
expr: (key < 10)
@@ -553,11 +568,11 @@ STAGE PLANS:
PREHOOK: query: SELECT * FROM srcpart WHERE ds='2008-04-09' AND hr=12 AND key < 10 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-PREHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-14-40_408_253015798760330492/-mr-10000
+PREHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-10-13_765_2757201616794436574/-mr-10000
POSTHOOK: query: SELECT * FROM srcpart WHERE ds='2008-04-09' AND hr=12 AND key < 10 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
-POSTHOOK: Output: file:/var/folders/D-/D-v4r3JYHU4wWmtdk5+mbU+++TI/-Tmp-/rmelick/hive_2011-04-21_20-14-40_408_253015798760330492/-mr-10000
+POSTHOOK: Output: file:/var/folders/5V/5V4Zq77qGD4aSK9m8V3frVsFdRU/-Tmp-/salbiz/hive_2011-06-15_16-10-13_765_2757201616794436574/-mr-10000
POSTHOOK: Lineage: default__src_src_index__._bucketname SIMPLE [(src)src.FieldSchema(name:INPUT__FILE__NAME, type:string, comment:), ]
POSTHOOK: Lineage: default__src_src_index__._offsets EXPRESSION [(src)src.FieldSchema(name:BLOCK__OFFSET__INSIDE__FILE, type:bigint, comment:), ]
POSTHOOK: Lineage: default__src_src_index__.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]